diff --git a/Cargo.lock b/Cargo.lock index 026786378d5a..991541ee84ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -322,6 +322,20 @@ dependencies = [ "num-traits", ] +[[package]] +name = "aquamarine" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1da02abba9f9063d786eab1509833ebb2fac0f966862ca59439c76b9c566760" +dependencies = [ + "include_dir", + "itertools 0.10.5", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "aquamarine" version = "0.5.0" @@ -4055,7 +4069,7 @@ dependencies = [ "cumulus-primitives-core", "cumulus-primitives-proof-size-hostfunction", "cumulus-test-runtime", - "docify", + "docify 0.2.7", "frame-support", "frame-system", "log", @@ -4736,13 +4750,39 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "docify" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af1b04e6ef3d21119d3eb7b032bca17f99fe041e9c072f30f32cc0e1a2b1f3c4" +dependencies = [ + "docify_macros 0.1.16", +] + [[package]] name = "docify" version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cc4fd38aaa9fb98ac70794c82a00360d1e165a87fbf96a8a91f9dfc602aaee2" dependencies = [ - "docify_macros", + "docify_macros 0.2.7", +] + +[[package]] +name = "docify_macros" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b5610df7f2acf89a1bb5d1a66ae56b1c7fcdcfe3948856fb3ace3f644d70eb7" +dependencies = [ + "common-path", + "derive-syn-parse", + "lazy_static", + "proc-macro2", + "quote", + "regex", + "syn 2.0.50", + "termcolor", + "walkdir", ] [[package]] @@ -5451,7 +5491,7 @@ checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" name = "frame" version = "0.0.1-dev" dependencies = [ - "docify", + "docify 0.2.7", "frame-executive", "frame-support", "frame-system", @@ -5619,6 +5659,7 @@ dependencies = [ name = "frame-executive" version = "28.0.0" dependencies = [ + "aquamarine 0.3.3", "array-bytes 6.1.0", "frame-support", "frame-system", @@ -5675,11 +5716,11 @@ dependencies = [ name = "frame-support" version = "28.0.0" dependencies = [ - "aquamarine", + "aquamarine 0.5.0", "array-bytes 6.1.0", "assert_matches", "bitflags 1.3.2", - "docify", + "docify 0.2.7", "environmental", "frame-metadata", "frame-support-procedural", @@ -5822,7 +5863,7 @@ version = "28.0.0" dependencies = [ "cfg-if", "criterion 0.4.0", - "docify", + "docify 0.2.7", "frame-support", "log", "parity-scale-codec", @@ -7081,6 +7122,7 @@ dependencies = [ "pallet-lottery", "pallet-membership", "pallet-message-queue", + "pallet-migrations", "pallet-mixnet", "pallet-mmr", "pallet-multisig", @@ -9301,8 +9343,8 @@ dependencies = [ name = "pallet-bags-list" version = "27.0.0" dependencies = [ - "aquamarine", - "docify", + "aquamarine 0.5.0", + "docify 0.2.7", "frame-benchmarking", "frame-election-provider-support", "frame-support", @@ -9350,7 +9392,7 @@ dependencies = [ name = "pallet-balances" version = "28.0.0" dependencies = [ - "docify", + "docify 0.2.7", "frame-benchmarking", "frame-support", "frame-system", @@ -9969,7 +10011,7 @@ dependencies = [ name = "pallet-example-single-block-migrations" version = "0.0.1" dependencies = [ - "docify", + "docify 0.2.7", "frame-executive", "frame-support", "frame-system", @@ -10035,7 +10077,7 @@ dependencies = [ name = "pallet-fast-unstake" version = "27.0.0" dependencies = [ - "docify", + "docify 0.2.7", "frame-benchmarking", "frame-election-provider-support", "frame-support", @@ -10232,6 +10274,30 @@ dependencies = [ "sp-weights", ] +[[package]] +name = "pallet-migrations" +version = "1.0.0" +dependencies = [ + "docify 0.1.16", + "frame-benchmarking", + "frame-executive", + "frame-support", + "frame-system", + "impl-trait-for-tuples", + "log", + "parity-scale-codec", + "pretty_assertions", + "scale-info", + "sp-api", + "sp-block-builder", + "sp-core", + "sp-io", + "sp-runtime", + "sp-std 14.0.0", + "sp-tracing 16.0.0", + "sp-version", +] + [[package]] name = "pallet-mixnet" version = "0.4.0" @@ -10507,7 +10573,7 @@ dependencies = [ name = "pallet-paged-list" version = "0.6.0" dependencies = [ - "docify", + "docify 0.2.7", "frame-benchmarking", "frame-support", "frame-system", @@ -10550,7 +10616,7 @@ dependencies = [ name = "pallet-parameters" version = "0.0.1" dependencies = [ - "docify", + "docify 0.2.7", "frame-benchmarking", "frame-support", "frame-system", @@ -10711,7 +10777,7 @@ dependencies = [ name = "pallet-safe-mode" version = "9.0.0" dependencies = [ - "docify", + "docify 0.2.7", "frame-benchmarking", "frame-support", "frame-system", @@ -10768,7 +10834,7 @@ dependencies = [ name = "pallet-scheduler" version = "29.0.0" dependencies = [ - "docify", + "docify 0.2.7", "frame-benchmarking", "frame-support", "frame-system", @@ -10981,7 +11047,7 @@ dependencies = [ name = "pallet-sudo" version = "28.0.0" dependencies = [ - "docify", + "docify 0.2.7", "frame-benchmarking", "frame-support", "frame-system", @@ -11012,7 +11078,7 @@ dependencies = [ name = "pallet-timestamp" version = "27.0.0" dependencies = [ - "docify", + "docify 0.2.7", "frame-benchmarking", "frame-support", "frame-system", @@ -11116,7 +11182,7 @@ dependencies = [ name = "pallet-treasury" version = "27.0.0" dependencies = [ - "docify", + "docify 0.2.7", "frame-benchmarking", "frame-support", "frame-system", @@ -11136,7 +11202,7 @@ dependencies = [ name = "pallet-tx-pause" version = "9.0.0" dependencies = [ - "docify", + "docify 0.2.7", "frame-benchmarking", "frame-support", "frame-system", @@ -13404,7 +13470,7 @@ version = "0.0.1" dependencies = [ "cumulus-pallet-aura-ext", "cumulus-pallet-parachain-system", - "docify", + "docify 0.2.7", "frame", "frame-executive", "frame-support", @@ -13416,6 +13482,7 @@ dependencies = [ "pallet-collective", "pallet-default-config-example", "pallet-democracy", + "pallet-example-offchain-worker", "pallet-example-single-block-migrations", "pallet-examples", "pallet-multisig", @@ -13442,6 +13509,7 @@ dependencies = [ "sp-core", "sp-io", "sp-keyring", + "sp-offchain", "sp-runtime", "sp-version", "staging-chain-spec-builder", @@ -13458,6 +13526,7 @@ version = "7.0.0" dependencies = [ "assert_matches", "async-trait", + "bitvec", "env_logger 0.9.3", "frame-benchmarking", "frame-benchmarking-cli", @@ -15663,7 +15732,7 @@ name = "sc-chain-spec" version = "27.0.0" dependencies = [ "array-bytes 6.1.0", - "docify", + "docify 0.2.7", "log", "memmap2 0.9.3", "parity-scale-codec", @@ -17854,7 +17923,7 @@ dependencies = [ [[package]] name = "snowbridge-beacon-primitives" -version = "0.0.0" +version = "0.2.0" dependencies = [ "byte-slice-cast", "frame-support", @@ -17878,7 +17947,7 @@ dependencies = [ [[package]] name = "snowbridge-core" -version = "0.0.0" +version = "0.2.0" dependencies = [ "ethabi-decode", "frame-support", @@ -17901,7 +17970,7 @@ dependencies = [ [[package]] name = "snowbridge-ethereum" -version = "0.1.0" +version = "0.3.0" dependencies = [ "ethabi-decode", "ethbloom", @@ -17940,7 +18009,7 @@ dependencies = [ [[package]] name = "snowbridge-outbound-queue-merkle-tree" -version = "0.1.1" +version = "0.3.0" dependencies = [ "array-bytes 4.2.0", "env_logger 0.9.3", @@ -17955,7 +18024,7 @@ dependencies = [ [[package]] name = "snowbridge-outbound-queue-runtime-api" -version = "0.0.0" +version = "0.2.0" dependencies = [ "frame-support", "parity-scale-codec", @@ -17969,7 +18038,7 @@ dependencies = [ [[package]] name = "snowbridge-pallet-ethereum-client" -version = "0.0.0" +version = "0.2.0" dependencies = [ "bp-runtime", "byte-slice-cast", @@ -18015,7 +18084,7 @@ dependencies = [ [[package]] name = "snowbridge-pallet-inbound-queue" -version = "0.0.0" +version = "0.2.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -18048,7 +18117,7 @@ dependencies = [ [[package]] name = "snowbridge-pallet-inbound-queue-fixtures" -version = "0.9.0" +version = "0.10.0" dependencies = [ "frame-benchmarking", "frame-support", @@ -18062,7 +18131,7 @@ dependencies = [ [[package]] name = "snowbridge-pallet-outbound-queue" -version = "0.0.0" +version = "0.2.0" dependencies = [ "bridge-hub-common", "ethabi-decode", @@ -18087,7 +18156,7 @@ dependencies = [ [[package]] name = "snowbridge-pallet-system" -version = "0.0.0" +version = "0.2.0" dependencies = [ "ethabi-decode", "frame-benchmarking", @@ -18115,7 +18184,7 @@ dependencies = [ [[package]] name = "snowbridge-router-primitives" -version = "0.0.0" +version = "0.9.0" dependencies = [ "ethabi-decode", "frame-support", @@ -18138,7 +18207,7 @@ dependencies = [ [[package]] name = "snowbridge-runtime-common" -version = "0.0.0" +version = "0.2.0" dependencies = [ "frame-support", "frame-system", @@ -18154,7 +18223,7 @@ dependencies = [ [[package]] name = "snowbridge-runtime-test-common" -version = "0.0.0" +version = "0.2.0" dependencies = [ "assets-common", "bridge-hub-test-utils", @@ -18231,7 +18300,7 @@ dependencies = [ [[package]] name = "snowbridge-system-runtime-api" -version = "0.0.0" +version = "0.2.0" dependencies = [ "parity-scale-codec", "snowbridge-core", @@ -18929,7 +18998,7 @@ dependencies = [ name = "sp-runtime" version = "31.0.1" dependencies = [ - "docify", + "docify 0.2.7", "either", "hash256-std-hasher", "impl-trait-for-tuples", diff --git a/Cargo.toml b/Cargo.toml index 654367cc1e21..48aa25f5c5a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -352,6 +352,7 @@ members = [ "substrate/frame/membership", "substrate/frame/merkle-mountain-range", "substrate/frame/message-queue", + "substrate/frame/migrations", "substrate/frame/mixnet", "substrate/frame/multisig", "substrate/frame/nft-fractionalization", diff --git a/bridges/snowbridge/pallets/ethereum-client/Cargo.toml b/bridges/snowbridge/pallets/ethereum-client/Cargo.toml index 99b429053114..c8999633c97a 100644 --- a/bridges/snowbridge/pallets/ethereum-client/Cargo.toml +++ b/bridges/snowbridge/pallets/ethereum-client/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snowbridge-pallet-ethereum-client" description = "Snowbridge Ethereum Client Pallet" -version = "0.0.0" +version = "0.2.0" authors = ["Snowfork "] edition.workspace = true repository.workspace = true diff --git a/bridges/snowbridge/pallets/inbound-queue/Cargo.toml b/bridges/snowbridge/pallets/inbound-queue/Cargo.toml index c26ef90a22db..b850496cd4e1 100644 --- a/bridges/snowbridge/pallets/inbound-queue/Cargo.toml +++ b/bridges/snowbridge/pallets/inbound-queue/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snowbridge-pallet-inbound-queue" description = "Snowbridge Inbound Queue Pallet" -version = "0.0.0" +version = "0.2.0" authors = ["Snowfork "] edition.workspace = true repository.workspace = true diff --git a/bridges/snowbridge/pallets/inbound-queue/fixtures/Cargo.toml b/bridges/snowbridge/pallets/inbound-queue/fixtures/Cargo.toml index 61f1421e0567..64605a42f0d3 100644 --- a/bridges/snowbridge/pallets/inbound-queue/fixtures/Cargo.toml +++ b/bridges/snowbridge/pallets/inbound-queue/fixtures/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snowbridge-pallet-inbound-queue-fixtures" description = "Snowbridge Inbound Queue Test Fixtures" -version = "0.9.0" +version = "0.10.0" authors = ["Snowfork "] edition.workspace = true repository.workspace = true diff --git a/bridges/snowbridge/pallets/inbound-queue/src/mock.rs b/bridges/snowbridge/pallets/inbound-queue/src/mock.rs index 110f611c6766..749fb0367f33 100644 --- a/bridges/snowbridge/pallets/inbound-queue/src/mock.rs +++ b/bridges/snowbridge/pallets/inbound-queue/src/mock.rs @@ -3,7 +3,7 @@ use super::*; use frame_support::{ - parameter_types, + derive_impl, parameter_types, traits::{ConstU128, ConstU32, Everything}, weights::IdentityFee, }; @@ -47,10 +47,9 @@ parameter_types! { type Balance = u128; +#[derive_impl(frame_system::config_preludes::TestDefaultConfig as frame_system::DefaultConfig)] impl frame_system::Config for Test { type BaseCallFilter = Everything; - type BlockWeights = (); - type BlockLength = (); type RuntimeOrigin = RuntimeOrigin; type RuntimeCall = RuntimeCall; type RuntimeTask = RuntimeTask; @@ -60,16 +59,8 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type RuntimeEvent = RuntimeEvent; type BlockHashCount = BlockHashCount; - type DbWeight = (); - type Version = (); type PalletInfo = PalletInfo; type AccountData = pallet_balances::AccountData; - type OnNewAccount = (); - type OnKilledAccount = (); - type SystemWeightInfo = (); - type SS58Prefix = (); - type OnSetCode = (); - type MaxConsumers = frame_support::traits::ConstU32<16>; type Nonce = u64; type Block = Block; } diff --git a/bridges/snowbridge/pallets/outbound-queue/Cargo.toml b/bridges/snowbridge/pallets/outbound-queue/Cargo.toml index 40e57db4bbd9..f16a28cb1e45 100644 --- a/bridges/snowbridge/pallets/outbound-queue/Cargo.toml +++ b/bridges/snowbridge/pallets/outbound-queue/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snowbridge-pallet-outbound-queue" description = "Snowbridge Outbound Queue Pallet" -version = "0.0.0" +version = "0.2.0" authors = ["Snowfork "] edition.workspace = true repository.workspace = true diff --git a/bridges/snowbridge/pallets/outbound-queue/merkle-tree/Cargo.toml b/bridges/snowbridge/pallets/outbound-queue/merkle-tree/Cargo.toml index c185d5af7062..0606e9de3305 100644 --- a/bridges/snowbridge/pallets/outbound-queue/merkle-tree/Cargo.toml +++ b/bridges/snowbridge/pallets/outbound-queue/merkle-tree/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snowbridge-outbound-queue-merkle-tree" description = "Snowbridge Outbound Queue Merkle Tree" -version = "0.1.1" +version = "0.3.0" authors = ["Snowfork "] edition.workspace = true repository.workspace = true diff --git a/bridges/snowbridge/pallets/outbound-queue/runtime-api/Cargo.toml b/bridges/snowbridge/pallets/outbound-queue/runtime-api/Cargo.toml index 347b3bae493b..cb68fd0a250a 100644 --- a/bridges/snowbridge/pallets/outbound-queue/runtime-api/Cargo.toml +++ b/bridges/snowbridge/pallets/outbound-queue/runtime-api/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snowbridge-outbound-queue-runtime-api" description = "Snowbridge Outbound Queue Runtime API" -version = "0.0.0" +version = "0.2.0" authors = ["Snowfork "] edition.workspace = true repository.workspace = true diff --git a/bridges/snowbridge/pallets/outbound-queue/src/mock.rs b/bridges/snowbridge/pallets/outbound-queue/src/mock.rs index dd8fee4e2ed0..6e78fb446721 100644 --- a/bridges/snowbridge/pallets/outbound-queue/src/mock.rs +++ b/bridges/snowbridge/pallets/outbound-queue/src/mock.rs @@ -3,7 +3,7 @@ use super::*; use frame_support::{ - parameter_types, + derive_impl, parameter_types, traits::{Everything, Hooks}, weights::IdentityFee, }; @@ -37,10 +37,9 @@ parameter_types! { pub const BlockHashCount: u64 = 250; } +#[derive_impl(frame_system::config_preludes::TestDefaultConfig as frame_system::DefaultConfig)] impl frame_system::Config for Test { type BaseCallFilter = Everything; - type BlockWeights = (); - type BlockLength = (); type RuntimeOrigin = RuntimeOrigin; type RuntimeCall = RuntimeCall; type RuntimeTask = RuntimeTask; @@ -50,16 +49,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type RuntimeEvent = RuntimeEvent; type BlockHashCount = BlockHashCount; - type DbWeight = (); - type Version = (); type PalletInfo = PalletInfo; - type AccountData = (); - type OnNewAccount = (); - type OnKilledAccount = (); - type SystemWeightInfo = (); - type SS58Prefix = (); - type OnSetCode = (); - type MaxConsumers = frame_support::traits::ConstU32<16>; type Nonce = u64; type Block = Block; } diff --git a/bridges/snowbridge/pallets/system/Cargo.toml b/bridges/snowbridge/pallets/system/Cargo.toml index f6c642e7376f..5ad04290de04 100644 --- a/bridges/snowbridge/pallets/system/Cargo.toml +++ b/bridges/snowbridge/pallets/system/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snowbridge-pallet-system" description = "Snowbridge System Pallet" -version = "0.0.0" +version = "0.2.0" authors = ["Snowfork "] edition.workspace = true repository.workspace = true diff --git a/bridges/snowbridge/pallets/system/runtime-api/Cargo.toml b/bridges/snowbridge/pallets/system/runtime-api/Cargo.toml index 355d2d29147f..eb02ae1db529 100644 --- a/bridges/snowbridge/pallets/system/runtime-api/Cargo.toml +++ b/bridges/snowbridge/pallets/system/runtime-api/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snowbridge-system-runtime-api" description = "Snowbridge System Runtime API" -version = "0.0.0" +version = "0.2.0" authors = ["Snowfork "] edition.workspace = true repository.workspace = true diff --git a/bridges/snowbridge/pallets/system/src/mock.rs b/bridges/snowbridge/pallets/system/src/mock.rs index edc3f141b073..de2970dd550b 100644 --- a/bridges/snowbridge/pallets/system/src/mock.rs +++ b/bridges/snowbridge/pallets/system/src/mock.rs @@ -2,8 +2,8 @@ // SPDX-FileCopyrightText: 2023 Snowfork use crate as snowbridge_system; use frame_support::{ - parameter_types, - traits::{tokens::fungible::Mutate, ConstU128, ConstU16, ConstU64, ConstU8}, + derive_impl, parameter_types, + traits::{tokens::fungible::Mutate, ConstU128, ConstU64, ConstU8}, weights::IdentityFee, PalletId, }; @@ -95,11 +95,9 @@ frame_support::construct_runtime!( } ); +#[derive_impl(frame_system::config_preludes::TestDefaultConfig as frame_system::DefaultConfig)] impl frame_system::Config for Test { type BaseCallFilter = frame_support::traits::Everything; - type BlockWeights = (); - type BlockLength = (); - type DbWeight = (); type RuntimeOrigin = RuntimeOrigin; type RuntimeCall = RuntimeCall; type RuntimeTask = RuntimeTask; @@ -109,15 +107,8 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type RuntimeEvent = RuntimeEvent; type BlockHashCount = ConstU64<250>; - type Version = (); type PalletInfo = PalletInfo; type AccountData = pallet_balances::AccountData; - type OnNewAccount = (); - type OnKilledAccount = (); - type SystemWeightInfo = (); - type SS58Prefix = ConstU16<42>; - type OnSetCode = (); - type MaxConsumers = frame_support::traits::ConstU32<16>; type Nonce = u64; type Block = Block; } diff --git a/bridges/snowbridge/parachain/pallets/ethereum-beacon-client/src/mock.rs b/bridges/snowbridge/parachain/pallets/ethereum-beacon-client/src/mock.rs new file mode 100644 index 000000000000..77b5c1aa631d --- /dev/null +++ b/bridges/snowbridge/parachain/pallets/ethereum-beacon-client/src/mock.rs @@ -0,0 +1,259 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2023 Snowfork +use crate as ethereum_beacon_client; +use frame_support::parameter_types; +use pallet_timestamp; +use primitives::{Fork, ForkVersions}; +use sp_core::H256; +use sp_runtime::traits::{BlakeTwo256, IdentityLookup}; + +#[cfg(not(feature = "beacon-spec-mainnet"))] +pub mod minimal { + use super::*; + + use crate::config; + use frame_support::derive_impl; + use hex_literal::hex; + use primitives::CompactExecutionHeader; + use snowbridge_core::inbound::{Log, Proof}; + use sp_runtime::BuildStorage; + use std::{fs::File, path::PathBuf}; + + type Block = frame_system::mocking::MockBlock; + + frame_support::construct_runtime!( + pub enum Test { + System: frame_system::{Pallet, Call, Storage, Event}, + Timestamp: pallet_timestamp::{Pallet, Call, Storage, Inherent}, + EthereumBeaconClient: ethereum_beacon_client::{Pallet, Call, Storage, Event}, + } + ); + + parameter_types! { + pub const BlockHashCount: u64 = 250; + pub const SS58Prefix: u8 = 42; + } + + #[derive_impl(frame_system::config_preludes::TestDefaultConfig as frame_system::DefaultConfig)] + impl frame_system::Config for Test { + type BaseCallFilter = frame_support::traits::Everything; + type RuntimeOrigin = RuntimeOrigin; + type RuntimeCall = RuntimeCall; + type RuntimeTask = RuntimeTask; + type Hash = H256; + type Hashing = BlakeTwo256; + type AccountId = u64; + type Lookup = IdentityLookup; + type RuntimeEvent = RuntimeEvent; + type BlockHashCount = BlockHashCount; + type PalletInfo = PalletInfo; + type SS58Prefix = SS58Prefix; + type Nonce = u64; + type Block = Block; + } + + impl pallet_timestamp::Config for Test { + type Moment = u64; + type OnTimestampSet = (); + type MinimumPeriod = (); + type WeightInfo = (); + } + + parameter_types! { + pub const ExecutionHeadersPruneThreshold: u32 = 10; + pub const ChainForkVersions: ForkVersions = ForkVersions{ + genesis: Fork { + version: [0, 0, 0, 1], // 0x00000001 + epoch: 0, + }, + altair: Fork { + version: [1, 0, 0, 1], // 0x01000001 + epoch: 0, + }, + bellatrix: Fork { + version: [2, 0, 0, 1], // 0x02000001 + epoch: 0, + }, + capella: Fork { + version: [3, 0, 0, 1], // 0x03000001 + epoch: 0, + }, + }; + } + + impl ethereum_beacon_client::Config for Test { + type RuntimeEvent = RuntimeEvent; + type ForkVersions = ChainForkVersions; + type MaxExecutionHeadersToKeep = ExecutionHeadersPruneThreshold; + type WeightInfo = (); + } + + // Build genesis storage according to the mock runtime. + pub fn new_tester() -> sp_io::TestExternalities { + let t = frame_system::GenesisConfig::::default().build_storage().unwrap(); + let mut ext = sp_io::TestExternalities::new(t); + let _ = ext.execute_with(|| Timestamp::set(RuntimeOrigin::signed(1), 30_000)); + ext + } + + fn load_fixture(basename: &str) -> Result + where + T: for<'de> serde::Deserialize<'de>, + { + let filepath: PathBuf = + [env!("CARGO_MANIFEST_DIR"), "tests", "fixtures", basename].iter().collect(); + serde_json::from_reader(File::open(filepath).unwrap()) + } + + pub fn load_execution_header_update_fixture() -> primitives::ExecutionHeaderUpdate { + load_fixture("execution-header-update.minimal.json").unwrap() + } + + pub fn load_checkpoint_update_fixture( + ) -> primitives::CheckpointUpdate<{ config::SYNC_COMMITTEE_SIZE }> { + load_fixture("initial-checkpoint.minimal.json").unwrap() + } + + pub fn load_sync_committee_update_fixture( + ) -> primitives::Update<{ config::SYNC_COMMITTEE_SIZE }, { config::SYNC_COMMITTEE_BITS_SIZE }> { + load_fixture("sync-committee-update.minimal.json").unwrap() + } + + pub fn load_finalized_header_update_fixture( + ) -> primitives::Update<{ config::SYNC_COMMITTEE_SIZE }, { config::SYNC_COMMITTEE_BITS_SIZE }> { + load_fixture("finalized-header-update.minimal.json").unwrap() + } + + pub fn load_next_sync_committee_update_fixture( + ) -> primitives::Update<{ config::SYNC_COMMITTEE_SIZE }, { config::SYNC_COMMITTEE_BITS_SIZE }> { + load_fixture("next-sync-committee-update.minimal.json").unwrap() + } + + pub fn load_next_finalized_header_update_fixture( + ) -> primitives::Update<{ config::SYNC_COMMITTEE_SIZE }, { config::SYNC_COMMITTEE_BITS_SIZE }> { + load_fixture("next-finalized-header-update.minimal.json").unwrap() + } + + pub fn get_message_verification_payload() -> (Log, Proof) { + ( + Log { + address: hex!("ee9170abfbf9421ad6dd07f6bdec9d89f2b581e0").into(), + topics: vec![ + hex!("1b11dcf133cc240f682dab2d3a8e4cd35c5da8c9cf99adac4336f8512584c5ad").into(), + hex!("00000000000000000000000000000000000000000000000000000000000003e8").into(), + hex!("0000000000000000000000000000000000000000000000000000000000000001").into(), + ], + data: hex!("0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000004b000f000000000000000100d184c103f7acc340847eee82a0b909e3358bc28d440edffa1352b13227e8ee646f3ea37456dec701345772617070656420457468657210574554481235003511000000000000000000000000000000000000000000").into(), + }, + Proof { + block_hash: hex!("05aaa60b0f27cce9e71909508527264b77ee14da7b5bf915fcc4e32715333213").into(), + tx_index: 0, + data: (vec![ + hex!("cf0d1c1ba57d1e0edfb59786c7e30c2b7e12bd54612b00cd21c4eaeecedf44fb").to_vec(), + hex!("d21fc4f68ab05bc4dcb23c67008e92c4d466437cdd6ed7aad0c008944c185510").to_vec(), + hex!("b9890f91ca0d77aa2a4adfaf9b9e40c94cac9e638b6d9797923865872944b646").to_vec(), + ], vec![ + hex!("f90131a0b601337b3aa10a671caa724eba641e759399979856141d3aea6b6b4ac59b889ba00c7d5dd48be9060221a02fb8fa213860b4c50d47046c8fa65ffaba5737d569e0a094601b62a1086cd9c9cb71a7ebff9e718f3217fd6e837efe4246733c0a196f63a06a4b0dd0aefc37b3c77828c8f07d1b7a2455ceb5dbfd3c77d7d6aeeddc2f7e8ca0d6e8e23142cdd8ec219e1f5d8b56aa18e456702b195deeaa210327284d42ade4a08a313d4c87023005d1ab631bbfe3f5de1e405d0e66d0bef3e033f1e5711b5521a0bf09a5d9a48b10ade82b8d6a5362a15921c8b5228a3487479b467db97411d82fa0f95cccae2a7c572ef3c566503e30bac2b2feb2d2f26eebf6d870dcf7f8cf59cea0d21fc4f68ab05bc4dcb23c67008e92c4d466437cdd6ed7aad0c008944c1855108080808080808080").to_vec(), + hex!("f851a0b9890f91ca0d77aa2a4adfaf9b9e40c94cac9e638b6d9797923865872944b646a060a634b9280e3a23fb63375e7bbdd9ab07fd379ab6a67e2312bbc112195fa358808080808080808080808080808080").to_vec(), + hex!("f9030820b9030402f90300018301d6e2b9010000000000000800000000000020040008000000000000000000000000400000008000000000000000000000000000000000000000000000000000000000042010000000001000000000000000000000000000000000040000000000000000000000000000000000000000000000008000000000000000002000000000000000000000000200000000000000200000000000100000000040000001000200008000000000000200000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000000f901f5f87a942ffa5ecdbe006d30397c7636d3e015eee251369ff842a0c965575a00553e094ca7c5d14f02e107c258dda06867cbf9e0e69f80e71bbcc1a000000000000000000000000000000000000000000000000000000000000003e8a000000000000000000000000000000000000000000000000000000000000003e8f9011c94ee9170abfbf9421ad6dd07f6bdec9d89f2b581e0f863a01b11dcf133cc240f682dab2d3a8e4cd35c5da8c9cf99adac4336f8512584c5ada000000000000000000000000000000000000000000000000000000000000003e8a00000000000000000000000000000000000000000000000000000000000000001b8a00000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000004b000f000000000000000100d184c103f7acc340847eee82a0b909e3358bc28d440edffa1352b13227e8ee646f3ea37456dec701345772617070656420457468657210574554481235003511000000000000000000000000000000000000000000f858948cf6147918a5cbb672703f879f385036f8793a24e1a01449abf21e49fd025f33495e77f7b1461caefdd3d4bb646424a3f445c4576a5ba0000000000000000000000000440edffa1352b13227e8ee646f3ea37456dec701").to_vec(), + ]), + } + ) + } + + pub fn get_message_verification_header() -> CompactExecutionHeader { + CompactExecutionHeader { + parent_hash: hex!("04a7f6ab8282203562c62f38b0ab41d32aaebe2c7ea687702b463148a6429e04") + .into(), + block_number: 55, + state_root: hex!("894d968712976d613519f973a317cb0781c7b039c89f27ea2b7ca193f7befdb3") + .into(), + receipts_root: hex!("cf0d1c1ba57d1e0edfb59786c7e30c2b7e12bd54612b00cd21c4eaeecedf44fb") + .into(), + } + } +} + +#[cfg(feature = "beacon-spec-mainnet")] +pub mod mainnet { + use super::*; + use frame_support::derive_impl; + + type Block = frame_system::mocking::MockBlock; + use sp_runtime::BuildStorage; + + frame_support::construct_runtime!( + pub enum Test { + System: frame_system::{Pallet, Call, Storage, Event}, + Timestamp: pallet_timestamp::{Pallet, Call, Storage, Inherent}, + EthereumBeaconClient: ethereum_beacon_client::{Pallet, Call, Storage, Event}, + } + ); + + parameter_types! { + pub const BlockHashCount: u64 = 250; + pub const SS58Prefix: u8 = 42; + } + + #[derive_impl(frame_system::config_preludes::TestDefaultConfig as frame_system::DefaultConfig)] + impl frame_system::Config for Test { + type BaseCallFilter = frame_support::traits::Everything; + type RuntimeOrigin = RuntimeOrigin; + type RuntimeCall = RuntimeCall; + type RuntimeTask = RuntimeTask; + type Hash = H256; + type Hashing = BlakeTwo256; + type AccountId = u64; + type Lookup = IdentityLookup; + type RuntimeEvent = RuntimeEvent; + type BlockHashCount = BlockHashCount; + type PalletInfo = PalletInfo; + type SS58Prefix = SS58Prefix; + type Nonce = u64; + type Block = Block; + } + + impl pallet_timestamp::Config for Test { + type Moment = u64; + type OnTimestampSet = (); + type MinimumPeriod = (); + type WeightInfo = (); + } + + parameter_types! { + pub const ChainForkVersions: ForkVersions = ForkVersions{ + genesis: Fork { + version: [0, 0, 16, 32], // 0x00001020 + epoch: 0, + }, + altair: Fork { + version: [1, 0, 16, 32], // 0x01001020 + epoch: 36660, + }, + bellatrix: Fork { + version: [2, 0, 16, 32], // 0x02001020 + epoch: 112260, + }, + capella: Fork { + version: [3, 0, 16, 32], // 0x03001020 + epoch: 162304, + }, + }; + pub const ExecutionHeadersPruneThreshold: u32 = 10; + } + + impl ethereum_beacon_client::Config for Test { + type RuntimeEvent = RuntimeEvent; + type ForkVersions = ChainForkVersions; + type MaxExecutionHeadersToKeep = ExecutionHeadersPruneThreshold; + type WeightInfo = (); + } + + // Build genesis storage according to the mock runtime. + pub fn new_tester() -> sp_io::TestExternalities { + let t = frame_system::GenesisConfig::::default().build_storage().unwrap(); + let mut ext = sp_io::TestExternalities::new(t); + let _ = ext.execute_with(|| Timestamp::set(RuntimeOrigin::signed(1), 30_000)); + ext + } +} diff --git a/bridges/snowbridge/primitives/beacon/Cargo.toml b/bridges/snowbridge/primitives/beacon/Cargo.toml index e021bb010714..d181fa1d3945 100644 --- a/bridges/snowbridge/primitives/beacon/Cargo.toml +++ b/bridges/snowbridge/primitives/beacon/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snowbridge-beacon-primitives" description = "Snowbridge Beacon Primitives" -version = "0.0.0" +version = "0.2.0" authors = ["Snowfork "] edition.workspace = true repository.workspace = true diff --git a/bridges/snowbridge/primitives/core/Cargo.toml b/bridges/snowbridge/primitives/core/Cargo.toml index 090c48c403fe..9a299ad0ae92 100644 --- a/bridges/snowbridge/primitives/core/Cargo.toml +++ b/bridges/snowbridge/primitives/core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snowbridge-core" description = "Snowbridge Core" -version = "0.0.0" +version = "0.2.0" authors = ["Snowfork "] edition.workspace = true repository.workspace = true diff --git a/bridges/snowbridge/primitives/ethereum/Cargo.toml b/bridges/snowbridge/primitives/ethereum/Cargo.toml index 399139cef381..9fa725a6c056 100644 --- a/bridges/snowbridge/primitives/ethereum/Cargo.toml +++ b/bridges/snowbridge/primitives/ethereum/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snowbridge-ethereum" description = "Snowbridge Ethereum" -version = "0.1.0" +version = "0.3.0" authors = ["Snowfork "] edition.workspace = true repository.workspace = true diff --git a/bridges/snowbridge/primitives/router/Cargo.toml b/bridges/snowbridge/primitives/router/Cargo.toml index 750a2a73e637..ded773e0d389 100644 --- a/bridges/snowbridge/primitives/router/Cargo.toml +++ b/bridges/snowbridge/primitives/router/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snowbridge-router-primitives" description = "Snowbridge Router Primitives" -version = "0.0.0" +version = "0.9.0" authors = ["Snowfork "] edition.workspace = true repository.workspace = true diff --git a/bridges/snowbridge/runtime/runtime-common/Cargo.toml b/bridges/snowbridge/runtime/runtime-common/Cargo.toml index d4c86f8aa750..bf5e9a8832dc 100644 --- a/bridges/snowbridge/runtime/runtime-common/Cargo.toml +++ b/bridges/snowbridge/runtime/runtime-common/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snowbridge-runtime-common" description = "Snowbridge Runtime Common" -version = "0.0.0" +version = "0.2.0" authors = ["Snowfork "] edition.workspace = true repository.workspace = true diff --git a/bridges/snowbridge/runtime/test-common/Cargo.toml b/bridges/snowbridge/runtime/test-common/Cargo.toml index ec4466b34265..4e8b311cb978 100644 --- a/bridges/snowbridge/runtime/test-common/Cargo.toml +++ b/bridges/snowbridge/runtime/test-common/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snowbridge-runtime-test-common" description = "Snowbridge Runtime Tests" -version = "0.0.0" +version = "0.2.0" authors = ["Snowfork "] edition = "2021" license = "Apache-2.0" diff --git a/bridges/snowbridge/runtime/test-common/src/lib.rs b/bridges/snowbridge/runtime/test-common/src/lib.rs index 29b0e738c182..7455adf76170 100644 --- a/bridges/snowbridge/runtime/test-common/src/lib.rs +++ b/bridges/snowbridge/runtime/test-common/src/lib.rs @@ -40,6 +40,7 @@ where } pub fn send_transfer_token_message( + ethereum_chain_id: u64, assethub_parachain_id: u32, weth_contract_address: H160, destination_address: H160, @@ -89,7 +90,7 @@ where WithdrawAsset(Assets::from(vec![fee.clone()])), BuyExecution { fees: fee, weight_limit: Unlimited }, ExportMessage { - network: Ethereum { chain_id: 11155111 }, + network: Ethereum { chain_id: ethereum_chain_id }, destination: Here, xcm: inner_xcm, }, @@ -107,6 +108,7 @@ where } pub fn send_transfer_token_message_success( + ethereum_chain_id: u64, collator_session_key: CollatorSessionKeys, runtime_para_id: u32, assethub_parachain_id: u32, @@ -149,6 +151,7 @@ pub fn send_transfer_token_message_success( initial_fund::(assethub_parachain_id, 5_000_000_000_000); let outcome = send_transfer_token_message::( + ethereum_chain_id, assethub_parachain_id, weth_contract_address, destination_address, @@ -205,6 +208,7 @@ pub fn ethereum_outbound_queue_processes_messages_before_message_queue_works< XcmConfig, AllPalletsWithoutSystem, >( + ethereum_chain_id: u64, collator_session_key: CollatorSessionKeys, runtime_para_id: u32, assethub_parachain_id: u32, @@ -249,6 +253,7 @@ pub fn ethereum_outbound_queue_processes_messages_before_message_queue_works< initial_fund::(assethub_parachain_id, 5_000_000_000_000); let outcome = send_transfer_token_message::( + ethereum_chain_id, assethub_parachain_id, weth_contract_address, destination_address, @@ -290,6 +295,7 @@ pub fn ethereum_outbound_queue_processes_messages_before_message_queue_works< } pub fn send_unpaid_transfer_token_message( + ethereum_chain_id: u64, collator_session_key: CollatorSessionKeys, runtime_para_id: u32, assethub_parachain_id: u32, @@ -353,7 +359,7 @@ pub fn send_unpaid_transfer_token_message( let xcm = Xcm(vec![ UnpaidExecution { weight_limit: Unlimited, check_origin: None }, ExportMessage { - network: Ethereum { chain_id: 11155111 }, + network: Ethereum { chain_id: ethereum_chain_id }, destination: Here, xcm: inner_xcm, }, @@ -375,6 +381,7 @@ pub fn send_unpaid_transfer_token_message( #[allow(clippy::too_many_arguments)] pub fn send_transfer_token_message_failure( + ethereum_chain_id: u64, collator_session_key: CollatorSessionKeys, runtime_para_id: u32, assethub_parachain_id: u32, @@ -414,6 +421,7 @@ pub fn send_transfer_token_message_failure( initial_fund::(assethub_parachain_id, initial_amount); let outcome = send_transfer_token_message::( + ethereum_chain_id, assethub_parachain_id, weth_contract_address, destination_address, diff --git a/bridges/snowbridge/scripts/contribute-upstream.sh b/bridges/snowbridge/scripts/contribute-upstream.sh index 8aa2d2a7035e..32005b770ecf 100755 --- a/bridges/snowbridge/scripts/contribute-upstream.sh +++ b/bridges/snowbridge/scripts/contribute-upstream.sh @@ -40,10 +40,12 @@ git checkout "$branch_name" # remove everything we think is not required for our needs rm -rf rust-toolchain.toml +rm -rf codecov.yml rm -rf $SNOWBRIDGE_FOLDER/.cargo rm -rf $SNOWBRIDGE_FOLDER/.github rm -rf $SNOWBRIDGE_FOLDER/SECURITY.md rm -rf $SNOWBRIDGE_FOLDER/.gitignore +rm -rf $SNOWBRIDGE_FOLDER/rustfmt.toml rm -rf $SNOWBRIDGE_FOLDER/templates rm -rf $SNOWBRIDGE_FOLDER/pallets/ethereum-client/fuzz diff --git a/cumulus/client/consensus/aura/src/lib.rs b/cumulus/client/consensus/aura/src/lib.rs index 8e4bc658e44b..ed6f5bdd4d69 100644 --- a/cumulus/client/consensus/aura/src/lib.rs +++ b/cumulus/client/consensus/aura/src/lib.rs @@ -54,7 +54,10 @@ use std::{ mod import_queue; pub use import_queue::{build_verifier, import_queue, BuildVerifierParams, ImportQueueParams}; -pub use sc_consensus_aura::{slot_duration, AuraVerifier, BuildAuraWorkerParams, SlotProportion}; +pub use sc_consensus_aura::{ + slot_duration, standalone::slot_duration_at, AuraVerifier, BuildAuraWorkerParams, + SlotProportion, +}; pub use sc_consensus_slots::InherentDataProviderExt; pub mod collator; diff --git a/cumulus/parachain-template/runtime/src/lib.rs b/cumulus/parachain-template/runtime/src/lib.rs index cee9b33bf37c..004a5d70ebc9 100644 --- a/cumulus/parachain-template/runtime/src/lib.rs +++ b/cumulus/parachain-template/runtime/src/lib.rs @@ -560,7 +560,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/assets/asset-hub-rococo/src/lib.rs b/cumulus/parachains/runtimes/assets/asset-hub-rococo/src/lib.rs index 6791dc4064ed..1c4f49fea0ed 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-rococo/src/lib.rs +++ b/cumulus/parachains/runtimes/assets/asset-hub-rococo/src/lib.rs @@ -1093,7 +1093,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/assets/asset-hub-westend/src/lib.rs b/cumulus/parachains/runtimes/assets/asset-hub-westend/src/lib.rs index 5352eeb1a1b7..e015c7bfca35 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-westend/src/lib.rs +++ b/cumulus/parachains/runtimes/assets/asset-hub-westend/src/lib.rs @@ -1124,7 +1124,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/lib.rs b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/lib.rs index db1dfbd45d63..0f6af1b51f1e 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/lib.rs +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/lib.rs @@ -816,7 +816,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/tests/snowbridge.rs b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/tests/snowbridge.rs index b9f43624b652..239bd946e759 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/tests/snowbridge.rs +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/tests/snowbridge.rs @@ -51,6 +51,7 @@ fn collator_session_keys() -> bridge_hub_test_utils::CollatorSessionKeys( + 11155111, collator_session_keys(), 1013, 1000, @@ -69,6 +70,7 @@ pub fn transfer_token_to_ethereum_works() { #[test] pub fn unpaid_transfer_token_to_ethereum_fails_with_barrier() { snowbridge_runtime_test_common::send_unpaid_transfer_token_message::( + 11155111, collator_session_keys(), 1013, 1000, @@ -80,6 +82,7 @@ pub fn unpaid_transfer_token_to_ethereum_fails_with_barrier() { #[test] pub fn transfer_token_to_ethereum_fee_not_enough() { snowbridge_runtime_test_common::send_transfer_token_message_failure::( + 11155111, collator_session_keys(), 1013, 1000, @@ -95,6 +98,7 @@ pub fn transfer_token_to_ethereum_fee_not_enough() { #[test] pub fn transfer_token_to_ethereum_insufficient_fund() { snowbridge_runtime_test_common::send_transfer_token_message_failure::( + 11155111, collator_session_keys(), 1013, 1000, @@ -146,6 +150,7 @@ pub fn ethereum_outbound_queue_processes_messages_before_message_queue_works() { XcmConfig, AllPalletsWithoutSystem, >( + 11155111, collator_session_keys(), 1013, 1000, diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/lib.rs b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/lib.rs index 1c8c9aa20755..206743ba6aca 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/lib.rs +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/lib.rs @@ -566,7 +566,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/collectives/collectives-westend/src/lib.rs b/cumulus/parachains/runtimes/collectives/collectives-westend/src/lib.rs index 36eb0e07f87c..3ed9d6e8e168 100644 --- a/cumulus/parachains/runtimes/collectives/collectives-westend/src/lib.rs +++ b/cumulus/parachains/runtimes/collectives/collectives-westend/src/lib.rs @@ -803,7 +803,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/contracts/contracts-rococo/src/lib.rs b/cumulus/parachains/runtimes/contracts/contracts-rococo/src/lib.rs index 541978098caa..c4df5b9a5652 100644 --- a/cumulus/parachains/runtimes/contracts/contracts-rococo/src/lib.rs +++ b/cumulus/parachains/runtimes/contracts/contracts-rococo/src/lib.rs @@ -466,7 +466,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/coretime/coretime-rococo/src/lib.rs b/cumulus/parachains/runtimes/coretime/coretime-rococo/src/lib.rs index fcd7576b1e17..0cdfebce742d 100644 --- a/cumulus/parachains/runtimes/coretime/coretime-rococo/src/lib.rs +++ b/cumulus/parachains/runtimes/coretime/coretime-rococo/src/lib.rs @@ -521,7 +521,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/coretime/coretime-westend/src/lib.rs b/cumulus/parachains/runtimes/coretime/coretime-westend/src/lib.rs index 80eb7863803b..7797329b5266 100644 --- a/cumulus/parachains/runtimes/coretime/coretime-westend/src/lib.rs +++ b/cumulus/parachains/runtimes/coretime/coretime-westend/src/lib.rs @@ -512,7 +512,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/glutton/glutton-westend/src/lib.rs b/cumulus/parachains/runtimes/glutton/glutton-westend/src/lib.rs index 10408aaf39a7..e09ad42ea9f2 100644 --- a/cumulus/parachains/runtimes/glutton/glutton-westend/src/lib.rs +++ b/cumulus/parachains/runtimes/glutton/glutton-westend/src/lib.rs @@ -332,7 +332,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/people/people-rococo/src/lib.rs b/cumulus/parachains/runtimes/people/people-rococo/src/lib.rs index 90c398917695..616da5519a5b 100644 --- a/cumulus/parachains/runtimes/people/people-rococo/src/lib.rs +++ b/cumulus/parachains/runtimes/people/people-rococo/src/lib.rs @@ -495,7 +495,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/people/people-westend/src/lib.rs b/cumulus/parachains/runtimes/people/people-westend/src/lib.rs index a904f7c3521b..9f572330cac8 100644 --- a/cumulus/parachains/runtimes/people/people-westend/src/lib.rs +++ b/cumulus/parachains/runtimes/people/people-westend/src/lib.rs @@ -495,7 +495,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/starters/seedling/src/lib.rs b/cumulus/parachains/runtimes/starters/seedling/src/lib.rs index ba077ef88794..4cc0a81ef49a 100644 --- a/cumulus/parachains/runtimes/starters/seedling/src/lib.rs +++ b/cumulus/parachains/runtimes/starters/seedling/src/lib.rs @@ -300,7 +300,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/starters/shell/src/lib.rs b/cumulus/parachains/runtimes/starters/shell/src/lib.rs index 457394760d98..829754731a42 100644 --- a/cumulus/parachains/runtimes/starters/shell/src/lib.rs +++ b/cumulus/parachains/runtimes/starters/shell/src/lib.rs @@ -357,7 +357,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/testing/penpal/src/lib.rs b/cumulus/parachains/runtimes/testing/penpal/src/lib.rs index bf8dcbc24c8d..0030287edb3b 100644 --- a/cumulus/parachains/runtimes/testing/penpal/src/lib.rs +++ b/cumulus/parachains/runtimes/testing/penpal/src/lib.rs @@ -699,7 +699,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/testing/rococo-parachain/src/lib.rs b/cumulus/parachains/runtimes/testing/rococo-parachain/src/lib.rs index 57969d9a4f18..b69c2341d6b0 100644 --- a/cumulus/parachains/runtimes/testing/rococo-parachain/src/lib.rs +++ b/cumulus/parachains/runtimes/testing/rococo-parachain/src/lib.rs @@ -689,7 +689,7 @@ impl_runtime_apis! { Executive::execute_block(block); } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/polkadot-parachain/src/fake_runtime_api/asset_hub_polkadot_aura.rs b/cumulus/polkadot-parachain/src/fake_runtime_api/asset_hub_polkadot_aura.rs index 76dd7347ccbc..7778d1bf7d2d 100644 --- a/cumulus/polkadot-parachain/src/fake_runtime_api/asset_hub_polkadot_aura.rs +++ b/cumulus/polkadot-parachain/src/fake_runtime_api/asset_hub_polkadot_aura.rs @@ -39,7 +39,7 @@ sp_api::impl_runtime_apis! { unimplemented!() } - fn initialize_block(_: &::Header) { + fn initialize_block(_: &::Header) -> sp_runtime::ExtrinsicInclusionMode { unimplemented!() } } diff --git a/cumulus/polkadot-parachain/src/fake_runtime_api/aura.rs b/cumulus/polkadot-parachain/src/fake_runtime_api/aura.rs index 0f01b85ebcf6..880f5d760c74 100644 --- a/cumulus/polkadot-parachain/src/fake_runtime_api/aura.rs +++ b/cumulus/polkadot-parachain/src/fake_runtime_api/aura.rs @@ -39,7 +39,7 @@ sp_api::impl_runtime_apis! { unimplemented!() } - fn initialize_block(_: &::Header) { + fn initialize_block(_: &::Header) -> sp_runtime::ExtrinsicInclusionMode { unimplemented!() } } diff --git a/cumulus/polkadot-parachain/src/service.rs b/cumulus/polkadot-parachain/src/service.rs index 4e06cd38f1d7..386551102e43 100644 --- a/cumulus/polkadot-parachain/src/service.rs +++ b/cumulus/polkadot-parachain/src/service.rs @@ -1251,28 +1251,33 @@ where <::Pair as Pair>::Signature: TryFrom> + std::hash::Hash + sp_runtime::traits::Member + Codec, { - let client2 = client.clone(); + let verifier_client = client.clone(); let aura_verifier = move || { - let slot_duration = cumulus_client_consensus_aura::slot_duration(&*client2).unwrap(); - Box::new(cumulus_client_consensus_aura::build_verifier::< ::Pair, _, _, _, >(cumulus_client_consensus_aura::BuildVerifierParams { - client: client2.clone(), - create_inherent_data_providers: move |_, _| async move { - let timestamp = sp_timestamp::InherentDataProvider::from_system_time(); - - let slot = - sp_consensus_aura::inherents::InherentDataProvider::from_timestamp_and_slot_duration( - *timestamp, - slot_duration, - ); - - Ok((slot, timestamp)) + client: verifier_client.clone(), + create_inherent_data_providers: move |parent_hash, _| { + let cidp_client = verifier_client.clone(); + async move { + let slot_duration = cumulus_client_consensus_aura::slot_duration_at( + &*cidp_client, + parent_hash, + )?; + let timestamp = sp_timestamp::InherentDataProvider::from_system_time(); + + let slot = + sp_consensus_aura::inherents::InherentDataProvider::from_timestamp_and_slot_duration( + *timestamp, + slot_duration, + ); + + Ok((slot, timestamp)) + } }, telemetry: telemetry_handle, })) as Box<_> diff --git a/cumulus/test/runtime/src/lib.rs b/cumulus/test/runtime/src/lib.rs index 5fb314109844..5ccec8983e91 100644 --- a/cumulus/test/runtime/src/lib.rs +++ b/cumulus/test/runtime/src/lib.rs @@ -377,7 +377,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/zombienet/tests/0002-pov_recovery.toml b/cumulus/zombienet/tests/0002-pov_recovery.toml index fe42fd4b2f66..15a61eba2a03 100644 --- a/cumulus/zombienet/tests/0002-pov_recovery.toml +++ b/cumulus/zombienet/tests/0002-pov_recovery.toml @@ -4,7 +4,7 @@ default_command = "polkadot" chain = "rococo-local" -[relaychain.genesis.runtimeGenesis.patch.configuration.config] +[relaychain.genesis.runtimeGenesis.patch.configuration.config.scheduler_params] # set parameters such that collators only connect to 1 validator as a backing group max_validators_per_core = 1 group_rotation_frequency = 100 # 10 mins diff --git a/docs/RELEASE.md b/docs/RELEASE.md index 6d681d78f367..e73be2779a99 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -18,10 +18,16 @@ Rococo. To easily refer to a release, it shall be named by its date in the form ## Crate -We try to follow [SemVer 2.0.0](https://semver.org/) as best as possible for versioning our crates. SemVer requires a -piece of software to first declare a public API. The public API of the Polkadot SDK is hereby declared as the sum of all -crates' public APIs. +We try to follow [SemVer 2.0.0](https://semver.org/) as best as possible for versioning our crates. The definitions of +`major`, `minor` and `patch` version for Rust crates are slightly altered from their standard for pre `1.0.0` versions. +Quoting [rust-lang.org](https://doc.rust-lang.org/cargo/reference/semver.html): +>Initial development releases starting with “0.y.z” can treat changes in “y” as a major release, and “z” as a minor +release. “0.0.z” releases are always major changes. This is because Cargo uses the convention that only changes in the +left-most non-zero component are considered incompatible. + +SemVer requires a piece of software to first declare a public API. The public API of the Polkadot SDK +is hereby declared as the sum of all crates' public APIs. Inductively, the public API of our library crates is declared as all public items that are neither: - Inside a `__private` module diff --git a/docs/sdk/Cargo.toml b/docs/sdk/Cargo.toml index b1b60a2d77db..735b3d7df61d 100644 --- a/docs/sdk/Cargo.toml +++ b/docs/sdk/Cargo.toml @@ -23,6 +23,7 @@ frame = { path = "../../substrate/frame", features = [ ] } pallet-examples = { path = "../../substrate/frame/examples" } pallet-default-config-example = { path = "../../substrate/frame/examples/default-config" } +pallet-example-offchain-worker = { path = "../../substrate/frame/examples/offchain-worker" } # How we build docs in rust-docs simple-mermaid = "0.1.1" @@ -38,7 +39,7 @@ frame-support = { path = "../../substrate/frame/support", default-features = fal frame-executive = { path = "../../substrate/frame/executive", default-features = false } pallet-example-single-block-migrations = { path = "../../substrate/frame/examples/single-block-migrations" } -# Substrate +# Substrate Client sc-network = { path = "../../substrate/client/network" } sc-rpc-api = { path = "../../substrate/client/rpc-api" } sc-rpc = { path = "../../substrate/client/rpc" } @@ -50,6 +51,7 @@ sc-consensus-grandpa = { path = "../../substrate/client/consensus/grandpa" } sc-consensus-beefy = { path = "../../substrate/client/consensus/beefy" } sc-consensus-manual-seal = { path = "../../substrate/client/consensus/manual-seal" } sc-consensus-pow = { path = "../../substrate/client/consensus/pow" } + substrate-wasm-builder = { path = "../../substrate/utils/wasm-builder" } # Cumulus @@ -78,6 +80,7 @@ sp-api = { path = "../../substrate/primitives/api" } sp-core = { path = "../../substrate/primitives/core" } sp-keyring = { path = "../../substrate/primitives/keyring" } sp-runtime = { path = "../../substrate/primitives/runtime" } +sp-offchain = { path = "../../substrate/primitives/offchain" } sp-version = { path = "../../substrate/primitives/version" } # XCM diff --git a/docs/sdk/headers/header.html b/docs/sdk/headers/header.html index 68dc5d5509e6..e28458c4ccc7 100644 --- a/docs/sdk/headers/header.html +++ b/docs/sdk/headers/header.html @@ -54,7 +54,7 @@ sidebarElems.style.display = 'none'; // Add click event listener to the button - expandButton.addEventListener('click', function() { + expandButton.addEventListener('click', function () { // Toggle the display of the '.sidebar-elems' if (sidebarElems.style.display === 'none') { sidebarElems.style.display = 'block'; @@ -72,6 +72,9 @@ if (!crate_name.textContent.startsWith("polkadot_sdk_docs")) { console.log("skipping -- not `polkadot_sdk_docs`"); return; + } else { + // insert class 'sdk-docs' to the body, so it enables the custom css rules. + document.body.classList.add("sdk-docs"); } createToC(); @@ -82,58 +85,60 @@ diff --git a/docs/sdk/headers/theme.css b/docs/sdk/headers/theme.css index bb9254ec4a82..a488e15c36b7 100644 --- a/docs/sdk/headers/theme.css +++ b/docs/sdk/headers/theme.css @@ -1,16 +1,17 @@ :root { - --polkadot-pink: #E6007A ; - --polkadot-green: #56F39A ; - --polkadot-lime: #D3FF33 ; - --polkadot-cyan: #00B2FF ; - --polkadot-purple: #552BBF ; - } - -body > nav.sidebar > div.sidebar-crate > a > img { - /* logo width, given that the sidebar width is 200px; */ - width: 190px; + --polkadot-pink: #E6007A; + --polkadot-green: #56F39A; + --polkadot-lime: #D3FF33; + --polkadot-cyan: #00B2FF; + --polkadot-purple: #552BBF; } -body nav.sidebar { - flex: 0 0 250px; +body.sdk-docs { + nav.sidebar>div.sidebar-crate>a>img { + width: 190px; + } + + nav.sidebar { + flex: 0 0 250px; + } } diff --git a/docs/sdk/src/reference_docs/development_environment_advice.rs b/docs/sdk/src/reference_docs/development_environment_advice.rs index 431769597936..21bbe78836c4 100644 --- a/docs/sdk/src/reference_docs/development_environment_advice.rs +++ b/docs/sdk/src/reference_docs/development_environment_advice.rs @@ -111,3 +111,74 @@ //! If you have a powerful remote server available, you may consider using //! [cargo-remote](https://github.com/sgeisler/cargo-remote) to execute cargo commands on it, //! freeing up local resources for other tasks like `rust-analyzer`. +//! +//! When using `cargo-remote`, you can configure your editor to perform the the typical +//! "check-on-save" remotely as well. The configuration for VSCode is as follows: +//! +//! ```json +//! { +//! "rust-analyzer.cargo.buildScripts.overrideCommand": [ +//! "cargo", +//! "remote", +//! "--build-env", +//! "SKIP_WASM_BUILD=1", +//! "--", +//! "check", +//! "--message-format=json", +//! "--all-targets", +//! "--all-features", +//! "--target-dir=target/rust-analyzer" +//! ], +//! "rust-analyzer.check.overrideCommand": [ +//! "cargo", +//! "remote", +//! "--build-env", +//! "SKIP_WASM_BUILD=1", +//! "--", +//! "check", +//! "--workspace", +//! "--message-format=json", +//! "--all-targets", +//! "--all-features", +//! "--target-dir=target/rust-analyzer" +//! ], +//! } +//! ``` +//! +//! //! and the same in Lua for `neovim/nvim-lspconfig`: +//! +//! ```lua +//! ["rust-analyzer"] = { +//! cargo = { +//! buildScripts = { +//! overrideCommand = { +//! "cargo", +//! "remote", +//! "--build-env", +//! "SKIP_WASM_BUILD=1", +//! "--", +//! "check", +//! "--message-format=json", +//! "--all-targets", +//! "--all-features", +//! "--target-dir=target/rust-analyzer" +//! }, +//! }, +//! check = { +//! overrideCommand = { +//! "cargo", +//! "remote", +//! "--build-env", +//! "SKIP_WASM_BUILD=1", +//! "--", +//! "check", +//! "--workspace", +//! "--message-format=json", +//! "--all-targets", +//! "--all-features", +//! "--target-dir=target/rust-analyzer" +//! }, +//! }, +//! }, +//! }, +//! ``` diff --git a/docs/sdk/src/reference_docs/frame_offchain_workers.rs b/docs/sdk/src/reference_docs/frame_offchain_workers.rs new file mode 100644 index 000000000000..7999707e5ee0 --- /dev/null +++ b/docs/sdk/src/reference_docs/frame_offchain_workers.rs @@ -0,0 +1,115 @@ +//! # Offchain Workers +//! +//! This reference document explains how offchain workers work in Substrate and FRAME. The main +//! focus is upon FRAME's implementation of this functionality. Nonetheless, offchain workers are a +//! Substrate-provided feature and can be used with possible alternatives to [`frame`] as well. +//! +//! Offchain workers are a commonly misunderstood topic, therefore we explain them bottom-up, +//! starting at the fundamentals and then describing the developer interface. +//! +//! ## Context +//! +//! Recall from [`crate::reference_docs::wasm_meta_protocol`] that the node and the runtime +//! communicate with one another via host functions and runtime APIs. Many of these interactions +//! contribute to the actual state transition of the blockchain. For example [`sp_api::Core`] is the +//! main runtime API that is called to execute new blocks. +//! +//! Offchain workers are in principle not different in any way: It is a runtime API exposed by the +//! wasm blob ([`sp_offchain::OffchainWorkerApi`]), and the node software calls into it when it +//! deems fit. But, crucially, this API call is different in that: +//! +//! 1. It can have no impact on the state ie. it is _OFF (the) CHAIN_. If any state is altered +//! during the execution of this API call, it is discarded. +//! 2. It has access to an extended set of host functions that allow the wasm blob to do more. For +//! example, call into HTTP requests. +//! +//! > The main way through which an offchain worker can interact with the state is by submitting an +//! > extrinsic to the chain. This is the ONLY way to alter the state from an offchain worker. +//! > [`pallet_example_offchain_worker`] provides an example of this. +//! +//! +//! Given the "Off Chain" nature of this API, it is important to remember that calling this API is +//! entirely optional. Some nodes might call into it, some might not, and it would have no impact on +//! the execution of your blockchain because no state is altered no matter the execution of the +//! offchain worker API. +//! +//! Substrate's CLI allows some degree of configuration about this, allowing node operators to +//! specify when they want to run the offchain worker API. See +//! [`sc_cli::RunCmd::offchain_worker_params`]. +//! +//! ## Nondeterministic Execution +//! +//! Needless to say, given the above description, the code in your offchain worker API can be +//! nondeterministic, as it is not part of the blockchain's STF, so it can be executed at unknown +//! times, by unknown nodes, and has no impact on the state. This is why an HTTP +//! ([`sp_runtime::offchain::http`]) API is readily provided to the offchain worker APIs. Because +//! there is no need for determinism in this context. +//! +//! > A common mistake here is for novice developers to see this HTTP API, and imagine that +//! > `polkadot-sdk` somehow magically solved the determinism in blockchains, and now a blockchain +//! > can make HTTP calls and it will all work. This is absolutely NOT the case. An HTTP call made +//! > by the offchain worker is non-deterministic by design. Blockchains can't and always won't be +//! > able to perform non-deterministic operations such as making HTTP calls to a foreign server. +//! +//! ## FRAME's API +//! +//! [`frame`] provides a simple API through which pallets can define offchain worker functions. This +//! is part of [`frame::traits::Hooks`], which is implemented as a part of +//! [`frame::pallet_macros::hooks`]. +//! +//! ``` +//! +//! #[frame::pallet] +//! pub mod pallet { +//! use frame::prelude::*; +//! +//! #[pallet::config] +//! pub trait Config: frame_system::Config {} +//! +//! #[pallet::pallet] +//! pub struct Pallet(_); +//! +//! #[pallet::hooks] +//! impl Hooks> for Pallet { +//! fn offchain_worker(block_number: BlockNumberFor) { +//! // ... +//! } +//! } +//! } +//! ``` +//! +//! Additionally, [`sp_runtime::offchain`] provides a set of utilities that can be used to moderate +//! the execution of offchain workers. +//! +//! ## Think Twice: Why Use Substrate's Offchain Workers? +//! +//! Consider the fact that in principle, an offchain worker code written using the above API is no +//! different than an equivalent written with an _actual offchain interaction library_, such as +//! [Polkadot-JS](https://polkadot.js.org/docs/), or any of the other ones listed [here](https://github.com/substrate-developer-hub/awesome-substrate?tab=readme-ov-file#client-libraries). +//! +//! They can both read from the state, and have no means of updating the state, other than the route +//! of submitting an extrinsic to the chain. Therefore, it is worth thinking twice before embedding +//! a logic as a part of Substrate's offchain worker API. Does it have to be there? can it not be a +//! simple, actual offchain application that lives outside of the chain's WASM blob? +//! +//! Some of the reasons why you might want to do the opposite, and actually embed an offchain worker +//! API into the WASM blob are: +//! +//! * Accessing the state is easier within the `offchain_worker` function, as it is already a part +//! of the runtime, and [`frame::pallet_macros::storage`] provides all the tools needed to read +//! the state. Other client libraries might provide varying degrees of capability here. +//! * It will be updated in synchrony with the runtime. A Substrate's offchain application is part +//! of the same WASM blob, and is therefore guaranteed to be up to date. +//! +//! For example, imagine you have modified a storage item to have a new type. This will possibly +//! require a [`crate::reference_docs::frame_runtime_upgrades_and_migrations`], and any offchain +//! code, such as a Polkadot-JS application, will have to be updated to reflect this change. Whereas +//! the WASM offchain worker code is guaranteed to already be updated, or else the runtime code will +//! not even compile. +//! +//! +//! ## Further References +//! +//! - +//! - +//! - [Offchain worker example](https://github.com/paritytech/polkadot-sdk/tree/master/substrate/frame/examples/offchain-worker) diff --git a/docs/sdk/src/reference_docs/mod.rs b/docs/sdk/src/reference_docs/mod.rs index b2c751420ce7..0bd204e4b6ab 100644 --- a/docs/sdk/src/reference_docs/mod.rs +++ b/docs/sdk/src/reference_docs/mod.rs @@ -99,3 +99,7 @@ pub mod frame_runtime_upgrades_and_migrations; /// light-node-first out of the box. // TODO: @jsdw @josepot https://github.com/paritytech/polkadot-sdk-docs/issues/68 pub mod light_nodes; + +/// Learn about the offchain workers, how they function, and how to use them, as provided by the +/// [`frame`] APIs. +pub mod frame_offchain_workers; diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index 456ae319787b..8cc16a6e1ec1 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -1253,13 +1253,20 @@ async fn handle_actions( Action::BecomeActive => { *mode = Mode::Active; - let messages = distribution_messages_for_activation( + let (messages, next_actions) = distribution_messages_for_activation( + ctx, overlayed_db, state, delayed_approvals_timers, - )?; + session_info_provider, + ) + .await?; ctx.send_messages(messages.into_iter()).await; + let next_actions: Vec = + next_actions.into_iter().map(|v| v.clone()).chain(actions_iter).collect(); + + actions_iter = next_actions.into_iter(); }, Action::Conclude => { conclude = true; @@ -1313,15 +1320,19 @@ fn get_assignment_core_indices( } } -fn distribution_messages_for_activation( +#[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] +async fn distribution_messages_for_activation( + ctx: &mut Context, db: &OverlayedBackend<'_, impl Backend>, state: &State, delayed_approvals_timers: &mut DelayedApprovalTimer, -) -> SubsystemResult> { + session_info_provider: &mut RuntimeInfo, +) -> SubsystemResult<(Vec, Vec)> { let all_blocks: Vec = db.load_all_blocks()?; let mut approval_meta = Vec::with_capacity(all_blocks.len()); let mut messages = Vec::new(); + let mut actions = Vec::new(); messages.push(ApprovalDistributionMessage::NewBlocks(Vec::new())); // dummy value. @@ -1396,16 +1407,60 @@ fn distribution_messages_for_activation( &claimed_core_indices, &block_entry, ) { - Ok(bitfield) => messages.push( - ApprovalDistributionMessage::DistributeAssignment( - IndirectAssignmentCertV2 { - block_hash, - validator: assignment.validator_index(), - cert: assignment.cert().clone(), - }, - bitfield, - ), - ), + Ok(bitfield) => { + gum::debug!( + target: LOG_TARGET, + candidate_hash = ?candidate_entry.candidate_receipt().hash(), + ?block_hash, + "Discovered, triggered assignment, not approved yet", + ); + + let indirect_cert = IndirectAssignmentCertV2 { + block_hash, + validator: assignment.validator_index(), + cert: assignment.cert().clone(), + }; + messages.push( + ApprovalDistributionMessage::DistributeAssignment( + indirect_cert.clone(), + bitfield.clone(), + ), + ); + + if !block_entry + .candidate_is_pending_signature(*candidate_hash) + { + let ExtendedSessionInfo { ref executor_params, .. } = + match get_extended_session_info( + session_info_provider, + ctx.sender(), + block_entry.block_hash(), + block_entry.session(), + ) + .await + { + Some(i) => i, + None => continue, + }; + + actions.push(Action::LaunchApproval { + claimed_candidate_indices: bitfield, + candidate_hash: candidate_entry + .candidate_receipt() + .hash(), + indirect_cert, + assignment_tranche: assignment.tranche(), + relay_block_hash: block_hash, + session: block_entry.session(), + executor_params: executor_params.clone(), + candidate: candidate_entry + .candidate_receipt() + .clone(), + backing_group: approval_entry.backing_group(), + distribute_assignment: false, + }); + } + }, Err(err) => { // Should never happen. If we fail here it means the // assignment is null (no cores claimed). @@ -1496,7 +1551,7 @@ fn distribution_messages_for_activation( } messages[0] = ApprovalDistributionMessage::NewBlocks(approval_meta); - Ok(messages) + Ok((messages, actions)) } // Handle an incoming signal from the overseer. Returns true if execution should conclude. diff --git a/polkadot/node/core/approval-voting/src/persisted_entries.rs b/polkadot/node/core/approval-voting/src/persisted_entries.rs index ef47bdb2213a..b924a1b52ccf 100644 --- a/polkadot/node/core/approval-voting/src/persisted_entries.rs +++ b/polkadot/node/core/approval-voting/src/persisted_entries.rs @@ -588,6 +588,13 @@ impl BlockEntry { !self.candidates_pending_signature.is_empty() } + /// Returns true if candidate hash is in the queue for a signature. + pub fn candidate_is_pending_signature(&self, candidate_hash: CandidateHash) -> bool { + self.candidates_pending_signature + .values() + .any(|context| context.candidate_hash == candidate_hash) + } + /// Candidate hashes for candidates pending signatures fn candidate_hashes_pending_signature(&self) -> Vec { self.candidates_pending_signature diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index 9220e84a2554..c9053232a4c8 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -78,6 +78,7 @@ struct TestSyncOracle { struct TestSyncOracleHandle { done_syncing_receiver: oneshot::Receiver<()>, + is_major_syncing: Arc, } impl TestSyncOracleHandle { @@ -108,8 +109,9 @@ impl SyncOracle for TestSyncOracle { fn make_sync_oracle(val: bool) -> (Box, TestSyncOracleHandle) { let (tx, rx) = oneshot::channel(); let flag = Arc::new(AtomicBool::new(val)); - let oracle = TestSyncOracle { flag, done_syncing_sender: Arc::new(Mutex::new(Some(tx))) }; - let handle = TestSyncOracleHandle { done_syncing_receiver: rx }; + let oracle = + TestSyncOracle { flag: flag.clone(), done_syncing_sender: Arc::new(Mutex::new(Some(tx))) }; + let handle = TestSyncOracleHandle { done_syncing_receiver: rx, is_major_syncing: flag }; (Box::new(oracle), handle) } @@ -465,6 +467,7 @@ struct HarnessConfigBuilder { clock: Option, backend: Option, assignment_criteria: Option>, + major_syncing: bool, } impl HarnessConfigBuilder { @@ -476,9 +479,19 @@ impl HarnessConfigBuilder { self } + pub fn major_syncing(&mut self, value: bool) -> &mut Self { + self.major_syncing = value; + self + } + + pub fn backend(&mut self, store: TestStore) -> &mut Self { + self.backend = Some(store); + self + } + pub fn build(&mut self) -> HarnessConfig { let (sync_oracle, sync_oracle_handle) = - self.sync_oracle.take().unwrap_or_else(|| make_sync_oracle(false)); + self.sync_oracle.take().unwrap_or_else(|| make_sync_oracle(self.major_syncing)); let assignment_criteria = self .assignment_criteria @@ -736,11 +749,13 @@ struct BlockConfig { slot: Slot, candidates: Option>, session_info: Option, + end_syncing: bool, } struct ChainBuilder { blocks_by_hash: HashMap, blocks_at_height: BTreeMap>, + is_major_syncing: Arc, } impl ChainBuilder { @@ -748,16 +763,28 @@ impl ChainBuilder { const GENESIS_PARENT_HASH: Hash = Hash::repeat_byte(0x00); pub fn new() -> Self { - let mut builder = - Self { blocks_by_hash: HashMap::new(), blocks_at_height: BTreeMap::new() }; + let mut builder = Self { + blocks_by_hash: HashMap::new(), + blocks_at_height: BTreeMap::new(), + is_major_syncing: Arc::new(AtomicBool::new(false)), + }; builder.add_block_inner( Self::GENESIS_HASH, Self::GENESIS_PARENT_HASH, 0, - BlockConfig { slot: Slot::from(0), candidates: None, session_info: None }, + BlockConfig { + slot: Slot::from(0), + candidates: None, + session_info: None, + end_syncing: false, + }, ); builder } + pub fn major_syncing(&mut self, major_syncing: Arc) -> &mut Self { + self.is_major_syncing = major_syncing; + self + } pub fn add_block( &mut self, @@ -808,8 +835,16 @@ impl ChainBuilder { } ancestry.reverse(); - import_block(overseer, ancestry.as_ref(), *number, block_config, false, i > 0) - .await; + import_block( + overseer, + ancestry.as_ref(), + *number, + block_config, + false, + i > 0, + self.is_major_syncing.clone(), + ) + .await; let _: Option<()> = future::pending().timeout(Duration::from_millis(100)).await; } } @@ -863,6 +898,7 @@ async fn import_block( config: &BlockConfig, gap: bool, fork: bool, + major_syncing: Arc, ) { let (new_head, new_header) = &hashes[hashes.len() - 1]; let candidates = config.candidates.clone().unwrap_or(vec![( @@ -891,6 +927,12 @@ async fn import_block( h_tx.send(Ok(Some(new_header.clone()))).unwrap(); } ); + + let is_major_syncing = major_syncing.load(Ordering::SeqCst); + if config.end_syncing { + major_syncing.store(false, Ordering::SeqCst); + } + if !fork { let mut _ancestry_step = 0; if gap { @@ -931,7 +973,7 @@ async fn import_block( } } - if number > 0 { + if number > 0 && !is_major_syncing { assert_matches!( overseer_recv(overseer).await, AllMessages::RuntimeApi( @@ -944,7 +986,6 @@ async fn import_block( c_tx.send(Ok(inclusion_events)).unwrap(); } ); - assert_matches!( overseer_recv(overseer).await, AllMessages::RuntimeApi( @@ -984,14 +1025,14 @@ async fn import_block( ); } - if number == 0 { + if number == 0 && !is_major_syncing { assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NewBlocks(v)) => { assert_eq!(v.len(), 0usize); } ); - } else { + } else if number > 0 && !is_major_syncing { if !fork { // SessionInfo won't be called for forks - it's already cached assert_matches!( @@ -1031,20 +1072,23 @@ async fn import_block( ); } - assert_matches!( - overseer_recv(overseer).await, - AllMessages::ApprovalDistribution( - ApprovalDistributionMessage::NewBlocks(mut approval_vec) - ) => { - assert_eq!(approval_vec.len(), 1); - let metadata = approval_vec.pop().unwrap(); - let hash = &hashes[number as usize]; - let parent_hash = &hashes[(number - 1) as usize]; - assert_eq!(metadata.hash, hash.0.clone()); - assert_eq!(metadata.parent_hash, parent_hash.0.clone()); - assert_eq!(metadata.slot, config.slot); - } - ); + if !is_major_syncing { + assert_matches!( + overseer_recv(overseer).await, + + AllMessages::ApprovalDistribution( + ApprovalDistributionMessage::NewBlocks(mut approval_vec) + ) => { + assert_eq!(approval_vec.len(), 1); + let metadata = approval_vec.pop().unwrap(); + let hash = &hashes[number as usize]; + let parent_hash = &hashes[(number - 1) as usize]; + assert_eq!(metadata.hash, hash.0.clone()); + assert_eq!(metadata.parent_hash, parent_hash.0.clone()); + assert_eq!(metadata.slot, config.slot); + } + ); + } } } @@ -1072,7 +1116,7 @@ fn subsystem_rejects_bad_assignment_ok_criteria() { block_hash, head, 1, - BlockConfig { slot, candidates: None, session_info: None }, + BlockConfig { slot, candidates: None, session_info: None, end_syncing: false }, ); builder.build(&mut virtual_overseer).await; @@ -1135,7 +1179,7 @@ fn subsystem_rejects_bad_assignment_err_criteria() { block_hash, head, 1, - BlockConfig { slot, candidates: None, session_info: None }, + BlockConfig { slot, candidates: None, session_info: None, end_syncing: false }, ); builder.build(&mut virtual_overseer).await; @@ -1240,6 +1284,7 @@ fn subsystem_rejects_approval_if_no_candidate_entry() { slot, candidates: Some(vec![(candidate_descriptor, CoreIndex(1), GroupIndex(1))]), session_info: None, + end_syncing: false, }, ); builder.build(&mut virtual_overseer).await; @@ -1345,7 +1390,12 @@ fn subsystem_rejects_approval_before_assignment() { block_hash, ChainBuilder::GENESIS_HASH, 1, - BlockConfig { slot: Slot::from(1), candidates: None, session_info: None }, + BlockConfig { + slot: Slot::from(1), + candidates: None, + session_info: None, + end_syncing: false, + }, ) .build(&mut virtual_overseer) .await; @@ -1398,7 +1448,12 @@ fn subsystem_rejects_assignment_in_future() { block_hash, ChainBuilder::GENESIS_HASH, 1, - BlockConfig { slot: Slot::from(0), candidates: None, session_info: None }, + BlockConfig { + slot: Slot::from(0), + candidates: None, + session_info: None, + end_syncing: false, + }, ) .build(&mut virtual_overseer) .await; @@ -1472,6 +1527,7 @@ fn subsystem_accepts_duplicate_assignment() { (candidate_receipt2, CoreIndex(1), GroupIndex(1)), ]), session_info: None, + end_syncing: false, }, ) .build(&mut virtual_overseer) @@ -1537,7 +1593,12 @@ fn subsystem_rejects_assignment_with_unknown_candidate() { block_hash, ChainBuilder::GENESIS_HASH, 1, - BlockConfig { slot: Slot::from(1), candidates: None, session_info: None }, + BlockConfig { + slot: Slot::from(1), + candidates: None, + session_info: None, + end_syncing: false, + }, ) .build(&mut virtual_overseer) .await; @@ -1582,7 +1643,12 @@ fn subsystem_rejects_oversized_bitfields() { block_hash, ChainBuilder::GENESIS_HASH, 1, - BlockConfig { slot: Slot::from(1), candidates: None, session_info: None }, + BlockConfig { + slot: Slot::from(1), + candidates: None, + session_info: None, + end_syncing: false, + }, ) .build(&mut virtual_overseer) .await; @@ -1650,7 +1716,12 @@ fn subsystem_accepts_and_imports_approval_after_assignment() { block_hash, ChainBuilder::GENESIS_HASH, 1, - BlockConfig { slot: Slot::from(1), candidates: None, session_info: None }, + BlockConfig { + slot: Slot::from(1), + candidates: None, + session_info: None, + end_syncing: false, + }, ) .build(&mut virtual_overseer) .await; @@ -1741,6 +1812,7 @@ fn subsystem_second_approval_import_only_schedules_wakeups() { slot: Slot::from(0), candidates: None, session_info: Some(session_info), + end_syncing: false, }, ) .build(&mut virtual_overseer) @@ -1824,7 +1896,12 @@ fn subsystem_assignment_import_updates_candidate_entry_and_schedules_wakeup() { block_hash, ChainBuilder::GENESIS_HASH, 1, - BlockConfig { slot: Slot::from(1), candidates: None, session_info: None }, + BlockConfig { + slot: Slot::from(1), + candidates: None, + session_info: None, + end_syncing: false, + }, ) .build(&mut virtual_overseer) .await; @@ -1873,7 +1950,12 @@ fn subsystem_process_wakeup_schedules_wakeup() { block_hash, ChainBuilder::GENESIS_HASH, 1, - BlockConfig { slot: Slot::from(1), candidates: None, session_info: None }, + BlockConfig { + slot: Slot::from(1), + candidates: None, + session_info: None, + end_syncing: false, + }, ) .build(&mut virtual_overseer) .await; @@ -1925,7 +2007,7 @@ fn linear_import_act_on_leaf() { hash, head, i, - BlockConfig { slot, candidates: None, session_info: None }, + BlockConfig { slot, candidates: None, session_info: None, end_syncing: false }, ); head = hash; } @@ -1983,7 +2065,7 @@ fn forkful_import_at_same_height_act_on_leaf() { hash, head, i, - BlockConfig { slot, candidates: None, session_info: None }, + BlockConfig { slot, candidates: None, session_info: None, end_syncing: false }, ); head = hash; } @@ -1997,7 +2079,7 @@ fn forkful_import_at_same_height_act_on_leaf() { hash, head, session, - BlockConfig { slot, candidates: None, session_info: None }, + BlockConfig { slot, candidates: None, session_info: None, end_syncing: false }, ); } builder.build(&mut virtual_overseer).await; @@ -2168,6 +2250,7 @@ fn import_checked_approval_updates_entries_and_schedules() { slot, candidates: Some(vec![(candidate_descriptor, CoreIndex(0), GroupIndex(0))]), session_info: Some(session_info), + end_syncing: false, }, ); builder.build(&mut virtual_overseer).await; @@ -2323,6 +2406,7 @@ fn subsystem_import_checked_approval_sets_one_block_bit_at_a_time() { (candidate_receipt2, CoreIndex(1), GroupIndex(1)), ]), session_info: Some(session_info), + end_syncing: false, }, ) .build(&mut virtual_overseer) @@ -2445,6 +2529,7 @@ fn approved_ancestor_test( slot: Slot::from(i as u64), candidates: Some(vec![(candidate_receipt, CoreIndex(0), GroupIndex(0))]), session_info: None, + end_syncing: false, }, ); } @@ -2623,13 +2708,19 @@ fn subsystem_validate_approvals_cache() { slot, candidates: candidates.clone(), session_info: Some(session_info.clone()), + end_syncing: false, }, ) .add_block( fork_block_hash, ChainBuilder::GENESIS_HASH, 1, - BlockConfig { slot, candidates, session_info: Some(session_info) }, + BlockConfig { + slot, + candidates, + session_info: Some(session_info), + end_syncing: false, + }, ) .build(&mut virtual_overseer) .await; @@ -2740,6 +2831,7 @@ fn subsystem_doesnt_distribute_duplicate_compact_assignments() { (candidate_receipt2.clone(), CoreIndex(1), GroupIndex(1)), ]), session_info: None, + end_syncing: false, }, ) .build(&mut virtual_overseer) @@ -2997,6 +3089,7 @@ where slot, candidates: Some(vec![(candidate_receipt, CoreIndex(0), GroupIndex(2))]), session_info: Some(session_info), + end_syncing: false, }, ) .build(&mut virtual_overseer) @@ -3314,6 +3407,7 @@ fn pre_covers_dont_stall_approval() { slot, candidates: Some(vec![(candidate_descriptor, CoreIndex(0), GroupIndex(0))]), session_info: Some(session_info), + end_syncing: false, }, ); builder.build(&mut virtual_overseer).await; @@ -3491,6 +3585,7 @@ fn waits_until_approving_assignments_are_old_enough() { slot, candidates: Some(vec![(candidate_descriptor, CoreIndex(0), GroupIndex(0))]), session_info: Some(session_info), + end_syncing: false, }, ); builder.build(&mut virtual_overseer).await; @@ -3705,6 +3800,7 @@ fn test_approval_is_sent_on_max_approval_coalesce_count() { slot, candidates: candidates.clone(), session_info: Some(session_info.clone()), + end_syncing: false, }, ) .build(&mut virtual_overseer) @@ -3943,8 +4039,7 @@ fn test_approval_is_sent_on_max_approval_coalesce_wait() { let store = config.backend(); test_harness(config, |test_harness| async move { - let TestHarness { mut virtual_overseer, clock, sync_oracle_handle: _sync_oracle_handle } = - test_harness; + let TestHarness { mut virtual_overseer, clock, sync_oracle_handle: _ } = test_harness; assert_matches!( overseer_recv(&mut virtual_overseer).await, @@ -4006,6 +4101,7 @@ fn test_approval_is_sent_on_max_approval_coalesce_wait() { slot, candidates: candidates.clone(), session_info: Some(session_info.clone()), + end_syncing: false, }, ) .build(&mut virtual_overseer) @@ -4040,3 +4136,569 @@ fn test_approval_is_sent_on_max_approval_coalesce_wait() { virtual_overseer }); } + +// Builds a chain with a fork where both relay blocks include the same candidate. +async fn build_chain_with_two_blocks_with_one_candidate_each( + block_hash1: Hash, + block_hash2: Hash, + slot: Slot, + sync_oracle_handle: TestSyncOracleHandle, + candidate_receipt: CandidateReceipt, +) -> (ChainBuilder, SessionInfo) { + let validators = vec![ + Sr25519Keyring::Alice, + Sr25519Keyring::Bob, + Sr25519Keyring::Charlie, + Sr25519Keyring::Dave, + Sr25519Keyring::Eve, + ]; + let session_info = SessionInfo { + validator_groups: IndexedVec::>::from(vec![ + vec![ValidatorIndex(0), ValidatorIndex(1)], + vec![ValidatorIndex(2)], + vec![ValidatorIndex(3), ValidatorIndex(4)], + ]), + ..session_info(&validators) + }; + + let candidates = Some(vec![(candidate_receipt.clone(), CoreIndex(0), GroupIndex(0))]); + let mut chain_builder = ChainBuilder::new(); + + chain_builder + .major_syncing(sync_oracle_handle.is_major_syncing.clone()) + .add_block( + block_hash1, + ChainBuilder::GENESIS_HASH, + 1, + BlockConfig { + slot, + candidates: candidates.clone(), + session_info: Some(session_info.clone()), + end_syncing: false, + }, + ) + .add_block( + block_hash2, + ChainBuilder::GENESIS_HASH, + 1, + BlockConfig { + slot, + candidates, + session_info: Some(session_info.clone()), + end_syncing: true, + }, + ); + (chain_builder, session_info) +} + +async fn setup_overseer_with_two_blocks_each_with_one_assignment_triggered( + virtual_overseer: &mut VirtualOverseer, + store: TestStore, + clock: &Box, + sync_oracle_handle: TestSyncOracleHandle, +) { + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::ChainApi(ChainApiMessage::FinalizedBlockNumber(rx)) => { + rx.send(Ok(0)).unwrap(); + } + ); + + let block_hash = Hash::repeat_byte(0x01); + let fork_block_hash = Hash::repeat_byte(0x02); + let candidate_commitments = CandidateCommitments::default(); + let mut candidate_receipt = dummy_candidate_receipt(block_hash); + candidate_receipt.commitments_hash = candidate_commitments.hash(); + let candidate_hash = candidate_receipt.hash(); + let slot = Slot::from(1); + let (chain_builder, _session_info) = build_chain_with_two_blocks_with_one_candidate_each( + block_hash, + fork_block_hash, + slot, + sync_oracle_handle, + candidate_receipt, + ) + .await; + chain_builder.build(virtual_overseer).await; + + assert!(!clock.inner.lock().current_wakeup_is(1)); + clock.inner.lock().wakeup_all(1); + + assert!(clock.inner.lock().current_wakeup_is(slot_to_tick(slot))); + clock.inner.lock().wakeup_all(slot_to_tick(slot)); + + futures_timer::Delay::new(Duration::from_millis(200)).await; + + clock.inner.lock().wakeup_all(slot_to_tick(slot + 2)); + + assert_eq!(clock.inner.lock().wakeups.len(), 0); + + futures_timer::Delay::new(Duration::from_millis(200)).await; + + let candidate_entry = store.load_candidate_entry(&candidate_hash).unwrap().unwrap(); + let our_assignment = + candidate_entry.approval_entry(&block_hash).unwrap().our_assignment().unwrap(); + assert!(our_assignment.triggered()); +} + +// Tests that for candidates that we did not approve yet, for which we triggered the assignment and +// the approval work we restart the work to approve it. +#[test] +fn subsystem_relaunches_approval_work_on_restart() { + let assignment_criteria = Box::new(MockAssignmentCriteria( + || { + let mut assignments = HashMap::new(); + let _ = assignments.insert( + CoreIndex(0), + approval_db::v2::OurAssignment { + cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }) + .into(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + + let _ = assignments.insert( + CoreIndex(0), + approval_db::v2::OurAssignment { + cert: garbage_assignment_cert_v2(AssignmentCertKindV2::RelayVRFModuloCompact { + core_bitfield: vec![CoreIndex(0), CoreIndex(1), CoreIndex(2)] + .try_into() + .unwrap(), + }), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + assignments + }, + |_| Ok(0), + )); + let config = HarnessConfigBuilder::default().assignment_criteria(assignment_criteria).build(); + let store = config.backend(); + let store_clone = config.backend(); + + test_harness(config, |test_harness| async move { + let TestHarness { mut virtual_overseer, clock, sync_oracle_handle } = test_harness; + + setup_overseer_with_two_blocks_each_with_one_assignment_triggered( + &mut virtual_overseer, + store, + &clock, + sync_oracle_handle, + ) + .await; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( + _, + _, + )) => { + } + ); + + recover_available_data(&mut virtual_overseer).await; + fetch_validation_code(&mut virtual_overseer).await; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( + _, + _ + )) => { + } + ); + + // Bail early after the assignment has been distributed but before we answer with the mocked + // approval from CandidateValidation. + virtual_overseer + }); + + // Restart a new approval voting subsystem with the same database and major syncing true untill + // the last leaf. + let config = HarnessConfigBuilder::default().backend(store_clone).major_syncing(true).build(); + + test_harness(config, |test_harness| async move { + let TestHarness { mut virtual_overseer, clock, sync_oracle_handle } = test_harness; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ChainApi(ChainApiMessage::FinalizedBlockNumber(rx)) => { + rx.send(Ok(0)).unwrap(); + } + ); + + let block_hash = Hash::repeat_byte(0x01); + let fork_block_hash = Hash::repeat_byte(0x02); + let candidate_commitments = CandidateCommitments::default(); + let mut candidate_receipt = dummy_candidate_receipt(block_hash); + candidate_receipt.commitments_hash = candidate_commitments.hash(); + let slot = Slot::from(1); + clock.inner.lock().set_tick(slot_to_tick(slot + 2)); + let (chain_builder, session_info) = build_chain_with_two_blocks_with_one_candidate_each( + block_hash, + fork_block_hash, + slot, + sync_oracle_handle, + candidate_receipt, + ) + .await; + + chain_builder.build(&mut virtual_overseer).await; + + futures_timer::Delay::new(Duration::from_millis(2000)).await; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request( + _, + RuntimeApiRequest::SessionInfo(_, si_tx), + ) + ) => { + si_tx.send(Ok(Some(session_info.clone()))).unwrap(); + } + ); + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request( + _, + RuntimeApiRequest::SessionExecutorParams(_, si_tx), + ) + ) => { + // Make sure all SessionExecutorParams calls are not made for the leaf (but for its relay parent) + si_tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(_, RuntimeApiRequest::NodeFeatures(_, si_tx), ) + ) => { + si_tx.send(Ok(NodeFeatures::EMPTY)).unwrap(); + } + ); + + // On major syncing ending Approval voting should send all the necessary messages for a + // candidate to be approved. + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NewBlocks( + _, + )) => { + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( + _, + _, + )) => { + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( + _, + _, + )) => { + } + ); + + // Guarantees the approval work has been relaunched. + recover_available_data(&mut virtual_overseer).await; + fetch_validation_code(&mut virtual_overseer).await; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive { + exec_kind, + response_sender, + .. + }) if exec_kind == PvfExecKind::Approval => { + response_sender.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) + .unwrap(); + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(_, sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: 1, + })); + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeApproval(_)) + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(_, sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: 1, + })); + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeApproval(_)) + ); + + // Assert that there are no more messages being sent by the subsystem + assert!(overseer_recv(&mut virtual_overseer).timeout(TIMEOUT / 2).await.is_none()); + + virtual_overseer + }); +} + +// Test that cached approvals, which are candidates that we approved but we didn't issue +// the signature yet because we want to coalesce it with more candidate are sent after restart. +#[test] +fn subsystem_sends_pending_approvals_on_approval_restart() { + let assignment_criteria = Box::new(MockAssignmentCriteria( + || { + let mut assignments = HashMap::new(); + let _ = assignments.insert( + CoreIndex(0), + approval_db::v2::OurAssignment { + cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }) + .into(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + + let _ = assignments.insert( + CoreIndex(0), + approval_db::v2::OurAssignment { + cert: garbage_assignment_cert_v2(AssignmentCertKindV2::RelayVRFModuloCompact { + core_bitfield: vec![CoreIndex(0), CoreIndex(1), CoreIndex(2)] + .try_into() + .unwrap(), + }), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + assignments + }, + |_| Ok(0), + )); + let config = HarnessConfigBuilder::default().assignment_criteria(assignment_criteria).build(); + let store = config.backend(); + let store_clone = config.backend(); + + test_harness(config, |test_harness| async move { + let TestHarness { mut virtual_overseer, clock, sync_oracle_handle } = test_harness; + + setup_overseer_with_two_blocks_each_with_one_assignment_triggered( + &mut virtual_overseer, + store, + &clock, + sync_oracle_handle, + ) + .await; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( + _, + _, + )) => { + } + ); + + recover_available_data(&mut virtual_overseer).await; + fetch_validation_code(&mut virtual_overseer).await; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( + _, + _ + )) => { + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive { + exec_kind, + response_sender, + .. + }) if exec_kind == PvfExecKind::Approval => { + response_sender.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) + .unwrap(); + } + ); + + // Configure a big coalesce number, so that the signature is cached instead of being sent to + // approval-distribution. + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(_, sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: 6, + })); + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(_, sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: 6, + })); + } + ); + + // Assert that there are no more messages being sent by the subsystem + assert!(overseer_recv(&mut virtual_overseer).timeout(TIMEOUT / 2).await.is_none()); + + virtual_overseer + }); + + let config = HarnessConfigBuilder::default().backend(store_clone).major_syncing(true).build(); + // On restart signatures should be sent to approval-distribution without relaunching the + // approval work. + test_harness(config, |test_harness| async move { + let TestHarness { mut virtual_overseer, clock, sync_oracle_handle } = test_harness; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ChainApi(ChainApiMessage::FinalizedBlockNumber(rx)) => { + rx.send(Ok(0)).unwrap(); + } + ); + + let block_hash = Hash::repeat_byte(0x01); + let fork_block_hash = Hash::repeat_byte(0x02); + let candidate_commitments = CandidateCommitments::default(); + let mut candidate_receipt = dummy_candidate_receipt(block_hash); + candidate_receipt.commitments_hash = candidate_commitments.hash(); + let slot = Slot::from(1); + + clock.inner.lock().set_tick(slot_to_tick(slot + 2)); + let (chain_builder, session_info) = build_chain_with_two_blocks_with_one_candidate_each( + block_hash, + fork_block_hash, + slot, + sync_oracle_handle, + candidate_receipt, + ) + .await; + chain_builder.build(&mut virtual_overseer).await; + + futures_timer::Delay::new(Duration::from_millis(2000)).await; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NewBlocks( + _, + )) => { + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( + _, + _, + )) => { + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( + _, + _, + )) => { + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(_, sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: 1, + })); + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request( + _, + RuntimeApiRequest::SessionInfo(_, si_tx), + ) + ) => { + si_tx.send(Ok(Some(session_info.clone()))).unwrap(); + } + ); + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request( + _, + RuntimeApiRequest::SessionExecutorParams(_, si_tx), + ) + ) => { + // Make sure all SessionExecutorParams calls are not made for the leaf (but for its relay parent) + si_tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(_, RuntimeApiRequest::NodeFeatures(_, si_tx), ) + ) => { + si_tx.send(Ok(NodeFeatures::EMPTY)).unwrap(); + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeApproval(_)) + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(_, sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: 1, + })); + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeApproval(_)) + ); + + // Assert that there are no more messages being sent by the subsystem + assert!(overseer_recv(&mut virtual_overseer).timeout(TIMEOUT / 2).await.is_none()); + + virtual_overseer + }); +} diff --git a/polkadot/node/core/candidate-validation/src/lib.rs b/polkadot/node/core/candidate-validation/src/lib.rs index bf6e09fd1b69..8237137fdca0 100644 --- a/polkadot/node/core/candidate-validation/src/lib.rs +++ b/polkadot/node/core/candidate-validation/src/lib.rs @@ -695,6 +695,8 @@ async fn validate_candidate_exhaustive( ))), Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError(err))) => Ok(ValidationResult::Invalid(InvalidCandidate::ExecutionError(err))), + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::RuntimeConstruction(err))) => + Ok(ValidationResult::Invalid(InvalidCandidate::ExecutionError(err))), Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousJobDeath(err))) => Ok(ValidationResult::Invalid(InvalidCandidate::ExecutionError(format!( @@ -780,40 +782,50 @@ trait ValidationBackend { return validation_result } + macro_rules! break_if_no_retries_left { + ($counter:ident) => { + if $counter > 0 { + $counter -= 1; + } else { + break + } + }; + } + // Allow limited retries for each kind of error. let mut num_death_retries_left = 1; let mut num_job_error_retries_left = 1; let mut num_internal_retries_left = 1; + let mut num_runtime_construction_retries_left = 1; loop { // Stop retrying if we exceeded the timeout. if total_time_start.elapsed() + retry_delay > exec_timeout { break } - + let mut retry_immediately = false; match validation_result { Err(ValidationError::PossiblyInvalid( PossiblyInvalidError::AmbiguousWorkerDeath | PossiblyInvalidError::AmbiguousJobDeath(_), - )) => - if num_death_retries_left > 0 { - num_death_retries_left -= 1; - } else { - break - }, + )) => break_if_no_retries_left!(num_death_retries_left), Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError(_))) => - if num_job_error_retries_left > 0 { - num_job_error_retries_left -= 1; - } else { - break - }, + break_if_no_retries_left!(num_job_error_retries_left), Err(ValidationError::Internal(_)) => - if num_internal_retries_left > 0 { - num_internal_retries_left -= 1; - } else { - break - }, + break_if_no_retries_left!(num_internal_retries_left), + + Err(ValidationError::PossiblyInvalid( + PossiblyInvalidError::RuntimeConstruction(_), + )) => { + break_if_no_retries_left!(num_runtime_construction_retries_left); + self.precheck_pvf(pvf.clone()).await?; + // In this case the error is deterministic + // And a retry forces the ValidationBackend + // to re-prepare the artifact so + // there is no need to wait before the retry + retry_immediately = true; + }, Ok(_) | Err(ValidationError::Invalid(_) | ValidationError::Preparation(_)) => break, } @@ -821,8 +833,11 @@ trait ValidationBackend { // If we got a possibly transient error, retry once after a brief delay, on the // assumption that the conditions that caused this error may have resolved on their own. { - // Wait a brief delay before retrying. - futures_timer::Delay::new(retry_delay).await; + // In case of many transient errors it is necessary to wait a little bit + // for the error to be probably resolved + if !retry_immediately { + futures_timer::Delay::new(retry_delay).await; + } let new_timeout = exec_timeout.saturating_sub(total_time_start.elapsed()); diff --git a/polkadot/node/core/pvf/common/src/error.rs b/polkadot/node/core/pvf/common/src/error.rs index f8faefc24e65..cf274044456f 100644 --- a/polkadot/node/core/pvf/common/src/error.rs +++ b/polkadot/node/core/pvf/common/src/error.rs @@ -16,6 +16,7 @@ use crate::prepare::{PrepareSuccess, PrepareWorkerSuccess}; use parity_scale_codec::{Decode, Encode}; +pub use sc_executor_common::error::Error as ExecuteError; /// Result of PVF preparation from a worker, with checksum of the compiled PVF and stats of the /// preparation if successful. diff --git a/polkadot/node/core/pvf/common/src/execute.rs b/polkadot/node/core/pvf/common/src/execute.rs index 6b3becf524d7..18c97b03cbcd 100644 --- a/polkadot/node/core/pvf/common/src/execute.rs +++ b/polkadot/node/core/pvf/common/src/execute.rs @@ -40,6 +40,9 @@ pub enum WorkerResponse { }, /// The candidate is invalid. InvalidCandidate(String), + /// Instantiation of the WASM module instance failed during an execution. + /// Possibly related to local issues or dirty node update. May be retried with re-preparation. + RuntimeConstruction(String), /// The job timed out. JobTimedOut, /// The job process has died. We must kill the worker just in case. @@ -68,6 +71,9 @@ pub enum JobResponse { /// The result of parachain validation. result_descriptor: ValidationResult, }, + /// A possibly transient runtime instantiation error happened during the execution; may be + /// retried with re-preparation + RuntimeConstruction(String), /// The candidate is invalid. InvalidCandidate(String), } @@ -81,6 +87,15 @@ impl JobResponse { Self::InvalidCandidate(format!("{}: {}", ctx, msg)) } } + + /// Creates a may retry response from a context `ctx` and a message `msg` (which can be empty). + pub fn runtime_construction(ctx: &'static str, msg: &str) -> Self { + if msg.is_empty() { + Self::RuntimeConstruction(ctx.to_string()) + } else { + Self::RuntimeConstruction(format!("{}: {}", ctx, msg)) + } + } } /// An unexpected error occurred in the execution job process. Because this comes from the job, diff --git a/polkadot/node/core/pvf/common/src/executor_interface.rs b/polkadot/node/core/pvf/common/src/executor_interface.rs index 4cd2f5c85eec..b69a87890f60 100644 --- a/polkadot/node/core/pvf/common/src/executor_interface.rs +++ b/polkadot/node/core/pvf/common/src/executor_interface.rs @@ -16,6 +16,7 @@ //! Interface to the Substrate Executor +use crate::error::ExecuteError; use polkadot_primitives::{ executor_params::{DEFAULT_LOGICAL_STACK_MAX, DEFAULT_NATIVE_STACK_MAX}, ExecutorParam, ExecutorParams, @@ -109,7 +110,7 @@ pub unsafe fn execute_artifact( compiled_artifact_blob: &[u8], executor_params: &ExecutorParams, params: &[u8], -) -> Result, String> { +) -> Result, ExecuteError> { let mut extensions = sp_externalities::Extensions::new(); extensions.register(sp_core::traits::ReadRuntimeVersionExt::new(ReadRuntimeVersion)); @@ -123,7 +124,6 @@ pub unsafe fn execute_artifact( Ok(Ok(ok)) => Ok(ok), Ok(Err(err)) | Err(err) => Err(err), } - .map_err(|err| format!("execute error: {:?}", err)) } /// Constructs the runtime for the given PVF, given the artifact bytes. diff --git a/polkadot/node/core/pvf/execute-worker/src/lib.rs b/polkadot/node/core/pvf/execute-worker/src/lib.rs index 0cfa5a786946..bd7e76010a6d 100644 --- a/polkadot/node/core/pvf/execute-worker/src/lib.rs +++ b/polkadot/node/core/pvf/execute-worker/src/lib.rs @@ -16,7 +16,9 @@ //! Contains the logic for executing PVFs. Used by the polkadot-execute-worker binary. -pub use polkadot_node_core_pvf_common::executor_interface::execute_artifact; +pub use polkadot_node_core_pvf_common::{ + error::ExecuteError, executor_interface::execute_artifact, +}; // NOTE: Initializing logging in e.g. tests will not have an effect in the workers, as they are // separate spawned processes. Run with e.g. `RUST_LOG=parachain::pvf-execute-worker=trace`. @@ -237,7 +239,9 @@ fn validate_using_artifact( // [`executor_interface::prepare`]. execute_artifact(compiled_artifact_blob, executor_params, params) } { - Err(err) => return JobResponse::format_invalid("execute", &err), + Err(ExecuteError::RuntimeConstruction(wasmerr)) => + return JobResponse::runtime_construction("execute", &wasmerr.to_string()), + Err(err) => return JobResponse::format_invalid("execute", &err.to_string()), Ok(d) => d, }; @@ -550,6 +554,8 @@ fn handle_parent_process( Ok(WorkerResponse::Ok { result_descriptor, duration: cpu_tv }) }, Ok(JobResponse::InvalidCandidate(err)) => Ok(WorkerResponse::InvalidCandidate(err)), + Ok(JobResponse::RuntimeConstruction(err)) => + Ok(WorkerResponse::RuntimeConstruction(err)), Err(job_error) => { gum::warn!( target: LOG_TARGET, diff --git a/polkadot/node/core/pvf/src/artifacts.rs b/polkadot/node/core/pvf/src/artifacts.rs index 78dfe71adadd..6288755526d4 100644 --- a/polkadot/node/core/pvf/src/artifacts.rs +++ b/polkadot/node/core/pvf/src/artifacts.rs @@ -238,6 +238,14 @@ impl Artifacts { .is_none()); } + /// Remove artifact by its id. + pub fn remove(&mut self, artifact_id: ArtifactId) -> Option<(ArtifactId, PathBuf)> { + self.inner.remove(&artifact_id).and_then(|state| match state { + ArtifactState::Prepared { path, .. } => Some((artifact_id, path)), + _ => None, + }) + } + /// Remove artifacts older than the given TTL and return id and path of the removed ones. pub fn prune(&mut self, artifact_ttl: Duration) -> Vec<(ArtifactId, PathBuf)> { let now = SystemTime::now(); diff --git a/polkadot/node/core/pvf/src/error.rs b/polkadot/node/core/pvf/src/error.rs index 80d41d5c64be..8dc96305eadb 100644 --- a/polkadot/node/core/pvf/src/error.rs +++ b/polkadot/node/core/pvf/src/error.rs @@ -86,6 +86,10 @@ pub enum PossiblyInvalidError { /// vote invalid. #[error("possibly invalid: job error: {0}")] JobError(String), + /// Instantiation of the WASM module instance failed during an execution. + /// Possibly related to local issues or dirty node update. May be retried with re-preparation. + #[error("possibly invalid: runtime construction: {0}")] + RuntimeConstruction(String), } impl From for ValidationError { diff --git a/polkadot/node/core/pvf/src/execute/mod.rs b/polkadot/node/core/pvf/src/execute/mod.rs index c6d9cf90fa28..365e98196cae 100644 --- a/polkadot/node/core/pvf/src/execute/mod.rs +++ b/polkadot/node/core/pvf/src/execute/mod.rs @@ -23,4 +23,4 @@ mod queue; mod worker_interface; -pub use queue::{start, PendingExecutionRequest, ToQueue}; +pub use queue::{start, FromQueue, PendingExecutionRequest, ToQueue}; diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index aa91d11781fc..bdc3c7327b06 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -25,7 +25,7 @@ use crate::{ InvalidCandidate, PossiblyInvalidError, ValidationError, LOG_TARGET, }; use futures::{ - channel::mpsc, + channel::{mpsc, oneshot}, future::BoxFuture, stream::{FuturesUnordered, StreamExt as _}, Future, FutureExt, @@ -54,6 +54,12 @@ pub enum ToQueue { Enqueue { artifact: ArtifactPathId, pending_execution_request: PendingExecutionRequest }, } +/// A response from queue. +#[derive(Debug)] +pub enum FromQueue { + RemoveArtifact { artifact: ArtifactId, reply_to: oneshot::Sender<()> }, +} + /// An execution request that should execute the PVF (known in the context) and send the results /// to the given result sender. #[derive(Debug)] @@ -137,6 +143,8 @@ struct Queue { /// The receiver that receives messages to the pool. to_queue_rx: mpsc::Receiver, + /// The sender to send messages back to validation host. + from_queue_tx: mpsc::UnboundedSender, // Some variables related to the current session. program_path: PathBuf, @@ -161,6 +169,7 @@ impl Queue { node_version: Option, security_status: SecurityStatus, to_queue_rx: mpsc::Receiver, + from_queue_tx: mpsc::UnboundedSender, ) -> Self { Self { metrics, @@ -170,6 +179,7 @@ impl Queue { node_version, security_status, to_queue_rx, + from_queue_tx, queue: VecDeque::new(), mux: Mux::new(), workers: Workers { @@ -301,7 +311,7 @@ async fn handle_mux(queue: &mut Queue, event: QueueEvent) { handle_worker_spawned(queue, idle, handle, job); }, QueueEvent::StartWork(worker, outcome, artifact_id, result_tx) => { - handle_job_finish(queue, worker, outcome, artifact_id, result_tx); + handle_job_finish(queue, worker, outcome, artifact_id, result_tx).await; }, } } @@ -327,42 +337,69 @@ fn handle_worker_spawned( /// If there are pending jobs in the queue, schedules the next of them onto the just freed up /// worker. Otherwise, puts back into the available workers list. -fn handle_job_finish( +async fn handle_job_finish( queue: &mut Queue, worker: Worker, outcome: Outcome, artifact_id: ArtifactId, result_tx: ResultSender, ) { - let (idle_worker, result, duration) = match outcome { + let (idle_worker, result, duration, sync_channel) = match outcome { Outcome::Ok { result_descriptor, duration, idle_worker } => { // TODO: propagate the soft timeout - (Some(idle_worker), Ok(result_descriptor), Some(duration)) + (Some(idle_worker), Ok(result_descriptor), Some(duration), None) }, Outcome::InvalidCandidate { err, idle_worker } => ( Some(idle_worker), Err(ValidationError::Invalid(InvalidCandidate::WorkerReportedInvalid(err))), None, + None, ), - Outcome::InternalError { err } => (None, Err(ValidationError::Internal(err)), None), + Outcome::RuntimeConstruction { err, idle_worker } => { + // The task for artifact removal is executed concurrently with + // the message to the host on the execution result. + let (result_tx, result_rx) = oneshot::channel(); + queue + .from_queue_tx + .unbounded_send(FromQueue::RemoveArtifact { + artifact: artifact_id.clone(), + reply_to: result_tx, + }) + .expect("from execute queue receiver is listened by the host; qed"); + ( + Some(idle_worker), + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::RuntimeConstruction( + err, + ))), + None, + Some(result_rx), + ) + }, + Outcome::InternalError { err } => (None, Err(ValidationError::Internal(err)), None, None), // Either the worker or the job timed out. Kill the worker in either case. Treated as // definitely-invalid, because if we timed out, there's no time left for a retry. Outcome::HardTimeout => - (None, Err(ValidationError::Invalid(InvalidCandidate::HardTimeout)), None), + (None, Err(ValidationError::Invalid(InvalidCandidate::HardTimeout)), None, None), // "Maybe invalid" errors (will retry). Outcome::WorkerIntfErr => ( None, Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath)), None, + None, ), Outcome::JobDied { err } => ( None, Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousJobDeath(err))), None, + None, + ), + Outcome::JobError { err } => ( + None, + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError(err))), + None, + None, ), - Outcome::JobError { err } => - (None, Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError(err))), None), }; queue.metrics.execute_finished(); @@ -386,6 +423,12 @@ fn handle_job_finish( ); } + if let Some(sync_channel) = sync_channel { + // err means the sender is dropped (the artifact is already removed from the cache) + // so that's legitimate to ignore the result + let _ = sync_channel.await; + } + // First we send the result. It may fail due to the other end of the channel being dropped, // that's legitimate and we don't treat that as an error. let _ = result_tx.send(result); @@ -521,8 +564,10 @@ pub fn start( spawn_timeout: Duration, node_version: Option, security_status: SecurityStatus, -) -> (mpsc::Sender, impl Future) { +) -> (mpsc::Sender, mpsc::UnboundedReceiver, impl Future) { let (to_queue_tx, to_queue_rx) = mpsc::channel(20); + let (from_queue_tx, from_queue_rx) = mpsc::unbounded(); + let run = Queue::new( metrics, program_path, @@ -532,7 +577,8 @@ pub fn start( node_version, security_status, to_queue_rx, + from_queue_tx, ) .run(); - (to_queue_tx, run) + (to_queue_tx, from_queue_rx, run) } diff --git a/polkadot/node/core/pvf/src/execute/worker_interface.rs b/polkadot/node/core/pvf/src/execute/worker_interface.rs index 9f7738f00e69..db81da118d7b 100644 --- a/polkadot/node/core/pvf/src/execute/worker_interface.rs +++ b/polkadot/node/core/pvf/src/execute/worker_interface.rs @@ -87,6 +87,10 @@ pub enum Outcome { /// a trap. Errors related to the preparation process are not expected to be encountered by the /// execution workers. InvalidCandidate { err: String, idle_worker: IdleWorker }, + /// The error is probably transient. It may be for example + /// because the artifact was prepared with a Wasmtime version different from the version + /// in the current execution environment. + RuntimeConstruction { err: String, idle_worker: IdleWorker }, /// The execution time exceeded the hard limit. The worker is terminated. HardTimeout, /// An I/O error happened during communication with the worker. This may mean that the worker @@ -193,6 +197,10 @@ pub async fn start_work( err, idle_worker: IdleWorker { stream, pid, worker_dir }, }, + WorkerResponse::RuntimeConstruction(err) => Outcome::RuntimeConstruction { + err, + idle_worker: IdleWorker { stream, pid, worker_dir }, + }, WorkerResponse::JobTimedOut => Outcome::HardTimeout, WorkerResponse::JobDied { err, job_pid: _ } => Outcome::JobDied { err }, WorkerResponse::JobError(err) => Outcome::JobError { err }, diff --git a/polkadot/node/core/pvf/src/host.rs b/polkadot/node/core/pvf/src/host.rs index ae9fdc7d2dea..8ec46f4b08f1 100644 --- a/polkadot/node/core/pvf/src/host.rs +++ b/polkadot/node/core/pvf/src/host.rs @@ -274,7 +274,7 @@ pub async fn start( from_prepare_pool, ); - let (to_execute_queue_tx, run_execute_queue) = execute::start( + let (to_execute_queue_tx, from_execute_queue_rx, run_execute_queue) = execute::start( metrics, config.execute_worker_program_path.to_owned(), config.cache_path.clone(), @@ -296,6 +296,7 @@ pub async fn start( to_prepare_queue_tx, from_prepare_queue_rx, to_execute_queue_tx, + from_execute_queue_rx, to_sweeper_tx, awaiting_prepare: AwaitingPrepare::default(), }) @@ -342,6 +343,8 @@ struct Inner { from_prepare_queue_rx: mpsc::UnboundedReceiver, to_execute_queue_tx: mpsc::Sender, + from_execute_queue_rx: mpsc::UnboundedReceiver, + to_sweeper_tx: mpsc::Sender, awaiting_prepare: AwaitingPrepare, @@ -358,6 +361,7 @@ async fn run( to_host_rx, from_prepare_queue_rx, mut to_prepare_queue_tx, + from_execute_queue_rx, mut to_execute_queue_tx, mut to_sweeper_tx, mut awaiting_prepare, @@ -384,10 +388,21 @@ async fn run( let mut to_host_rx = to_host_rx.fuse(); let mut from_prepare_queue_rx = from_prepare_queue_rx.fuse(); + let mut from_execute_queue_rx = from_execute_queue_rx.fuse(); loop { // biased to make it behave deterministically for tests. futures::select_biased! { + from_execute_queue_rx = from_execute_queue_rx.next() => { + let from_queue = break_if_fatal!(from_execute_queue_rx.ok_or(Fatal)); + let execute::FromQueue::RemoveArtifact { artifact, reply_to } = from_queue; + break_if_fatal!(handle_artifact_removal( + &mut to_sweeper_tx, + &mut artifacts, + artifact, + reply_to, + ).await); + }, () = cleanup_pulse.select_next_some() => { // `select_next_some` because we don't expect this to fail, but if it does, we // still don't fail. The trade-off is that the compiled cache will start growing @@ -861,6 +876,37 @@ async fn handle_cleanup_pulse( Ok(()) } +async fn handle_artifact_removal( + sweeper_tx: &mut mpsc::Sender, + artifacts: &mut Artifacts, + artifact_id: ArtifactId, + reply_to: oneshot::Sender<()>, +) -> Result<(), Fatal> { + let (artifact_id, path) = if let Some(artifact) = artifacts.remove(artifact_id) { + artifact + } else { + // if we haven't found the artifact by its id, + // it has been probably removed + // anyway with the randomness of the artifact name + // it is safe to ignore + return Ok(()); + }; + reply_to + .send(()) + .expect("the execute queue waits for the artifact remove confirmation; qed"); + // Thanks to the randomness of the artifact name (see + // `artifacts::generate_artifact_path`) there is no issue with any name conflict on + // future repreparation. + // So we can confirm the artifact removal already + gum::debug!( + target: LOG_TARGET, + validation_code_hash = ?artifact_id.code_hash, + "PVF pruning: pruning artifact by request from the execute queue", + ); + sweeper_tx.send(path).await.map_err(|_| Fatal)?; + Ok(()) +} + /// A simple task which sole purpose is to delete files thrown at it. async fn sweeper_task(mut sweeper_rx: mpsc::Receiver) { loop { @@ -968,6 +1014,8 @@ pub(crate) mod tests { to_prepare_queue_rx: mpsc::Receiver, from_prepare_queue_tx: mpsc::UnboundedSender, to_execute_queue_rx: mpsc::Receiver, + #[allow(unused)] + from_execute_queue_tx: mpsc::UnboundedSender, to_sweeper_rx: mpsc::Receiver, run: BoxFuture<'static, ()>, @@ -979,6 +1027,7 @@ pub(crate) mod tests { let (to_prepare_queue_tx, to_prepare_queue_rx) = mpsc::channel(10); let (from_prepare_queue_tx, from_prepare_queue_rx) = mpsc::unbounded(); let (to_execute_queue_tx, to_execute_queue_rx) = mpsc::channel(10); + let (from_execute_queue_tx, from_execute_queue_rx) = mpsc::unbounded(); let (to_sweeper_tx, to_sweeper_rx) = mpsc::channel(10); let run = run(Inner { @@ -989,6 +1038,7 @@ pub(crate) mod tests { to_prepare_queue_tx, from_prepare_queue_rx, to_execute_queue_tx, + from_execute_queue_rx, to_sweeper_tx, awaiting_prepare: AwaitingPrepare::default(), }) @@ -999,6 +1049,7 @@ pub(crate) mod tests { to_prepare_queue_rx, from_prepare_queue_tx, to_execute_queue_rx, + from_execute_queue_tx, to_sweeper_rx, run, } diff --git a/polkadot/node/core/pvf/tests/it/main.rs b/polkadot/node/core/pvf/tests/it/main.rs index bcc10749e746..cdfbcd8e5785 100644 --- a/polkadot/node/core/pvf/tests/it/main.rs +++ b/polkadot/node/core/pvf/tests/it/main.rs @@ -21,13 +21,14 @@ use parity_scale_codec::Encode as _; #[cfg(all(feature = "ci-only-tests", target_os = "linux"))] use polkadot_node_core_pvf::SecurityStatus; use polkadot_node_core_pvf::{ - start, testing::build_workers_and_get_paths, Config, InvalidCandidate, Metrics, PrepareError, - PrepareJobKind, PvfPrepData, ValidationError, ValidationHost, JOB_TIMEOUT_WALL_CLOCK_FACTOR, + start, testing::build_workers_and_get_paths, Config, InvalidCandidate, Metrics, + PossiblyInvalidError, PrepareError, PrepareJobKind, PvfPrepData, ValidationError, + ValidationHost, JOB_TIMEOUT_WALL_CLOCK_FACTOR, }; use polkadot_parachain_primitives::primitives::{BlockData, ValidationParams, ValidationResult}; use polkadot_primitives::{ExecutorParam, ExecutorParams}; -use std::time::Duration; +use std::{io::Write, time::Duration}; use tokio::sync::Mutex; mod adder; @@ -352,10 +353,80 @@ async fn deleting_prepared_artifact_does_not_dispute() { ) .await; - match result { - Err(ValidationError::Invalid(InvalidCandidate::HardTimeout)) => {}, - r => panic!("{:?}", r), + assert_matches!(result, Err(ValidationError::Invalid(InvalidCandidate::HardTimeout))); +} + +// Test that corruption of a prepared artifact does not lead to a dispute when we try to execute it. +#[tokio::test] +async fn corrupted_prepared_artifact_does_not_dispute() { + let host = TestHost::new().await; + let cache_dir = host.cache_dir.path(); + + let _stats = host.precheck_pvf(halt::wasm_binary_unwrap(), Default::default()).await.unwrap(); + + // Manually corrupting the prepared artifact from disk. The in-memory artifacts table won't + // change. + let artifact_path = { + // Get the artifact path (asserting it exists). + let mut cache_dir: Vec<_> = std::fs::read_dir(cache_dir).unwrap().collect(); + // Should contain the artifact and the worker dir. + assert_eq!(cache_dir.len(), 2); + let mut artifact_path = cache_dir.pop().unwrap().unwrap(); + if artifact_path.path().is_dir() { + artifact_path = cache_dir.pop().unwrap().unwrap(); + } + + // Corrupt the artifact. + let mut f = std::fs::OpenOptions::new() + .write(true) + .truncate(true) + .open(artifact_path.path()) + .unwrap(); + f.write_all(b"corrupted wasm").unwrap(); + f.flush().unwrap(); + artifact_path + }; + + assert!(artifact_path.path().exists()); + + // Try to validate, artifact should get removed because of the corruption. + let result = host + .validate_candidate( + halt::wasm_binary_unwrap(), + ValidationParams { + block_data: BlockData(Vec::new()), + parent_head: Default::default(), + relay_parent_number: 1, + relay_parent_storage_root: Default::default(), + }, + Default::default(), + ) + .await; + + assert_matches!( + result, + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::RuntimeConstruction(_))) + ); + + // because of RuntimeConstruction we may retry + host.precheck_pvf(halt::wasm_binary_unwrap(), Default::default()).await.unwrap(); + + // The actual artifact removal is done concurrently + // with sending of the result of the execution + // it is not a problem for further re-preparation as + // artifact filenames are random + for _ in 1..5 { + if !artifact_path.path().exists() { + break; + } + tokio::time::sleep(Duration::from_secs(1)).await; } + + assert!( + !artifact_path.path().exists(), + "the corrupted artifact ({}) should be deleted by the host", + artifact_path.path().display() + ); } #[tokio::test] diff --git a/polkadot/node/service/Cargo.toml b/polkadot/node/service/Cargo.toml index 8fd9f20b7bcf..734dcbdeb441 100644 --- a/polkadot/node/service/Cargo.toml +++ b/polkadot/node/service/Cargo.toml @@ -93,6 +93,7 @@ kvdb-rocksdb = { version = "0.19.0", optional = true } parity-db = { version = "0.4.12", optional = true } codec = { package = "parity-scale-codec", version = "3.6.1" } parking_lot = "0.12.1" +bitvec = { version = "1.0.1", optional = true } # Polkadot polkadot-core-primitives = { path = "../../core-primitives" } @@ -184,8 +185,8 @@ full-node = [ ] # Configure the native runtimes to use. -westend-native = ["westend-runtime", "westend-runtime-constants"] -rococo-native = ["rococo-runtime", "rococo-runtime-constants"] +westend-native = ["bitvec", "westend-runtime", "westend-runtime-constants"] +rococo-native = ["bitvec", "rococo-runtime", "rococo-runtime-constants"] runtime-benchmarks = [ "frame-benchmarking-cli/runtime-benchmarks", diff --git a/polkadot/node/service/src/chain_spec.rs b/polkadot/node/service/src/chain_spec.rs index af241d1cbc55..1c44b17b6fd2 100644 --- a/polkadot/node/service/src/chain_spec.rs +++ b/polkadot/node/service/src/chain_spec.rs @@ -24,6 +24,8 @@ use polkadot_primitives::{AccountId, AccountPublic, AssignmentId, ValidatorId}; use sp_authority_discovery::AuthorityId as AuthorityDiscoveryId; use sp_consensus_babe::AuthorityId as BabeId; +#[cfg(any(feature = "rococo-native", feature = "westend-native",))] +use polkadot_primitives::vstaging::SchedulerParams; #[cfg(feature = "rococo-native")] use rococo_runtime as rococo; #[cfg(feature = "rococo-native")] @@ -120,7 +122,9 @@ pub fn wococo_config() -> Result { fn default_parachains_host_configuration( ) -> polkadot_runtime_parachains::configuration::HostConfiguration { - use polkadot_primitives::{AsyncBackingParams, MAX_CODE_SIZE, MAX_POV_SIZE}; + use polkadot_primitives::{ + vstaging::node_features::FeatureIndex, AsyncBackingParams, MAX_CODE_SIZE, MAX_POV_SIZE, + }; polkadot_runtime_parachains::configuration::HostConfiguration { validation_upgrade_cooldown: 2u32, @@ -129,8 +133,6 @@ fn default_parachains_host_configuration( max_code_size: MAX_CODE_SIZE, max_pov_size: MAX_POV_SIZE, max_head_data_size: 32 * 1024, - group_rotation_frequency: 20, - paras_availability_period: 4, max_upward_queue_count: 8, max_upward_queue_size: 1024 * 1024, max_downward_message_size: 1024 * 1024, @@ -151,11 +153,19 @@ fn default_parachains_host_configuration( relay_vrf_modulo_samples: 2, zeroth_delay_tranche_width: 0, minimum_validation_upgrade_delay: 5, - scheduling_lookahead: 2, async_backing_params: AsyncBackingParams { max_candidate_depth: 3, allowed_ancestry_len: 2, }, + node_features: bitvec::vec::BitVec::from_element( + 1u8 << (FeatureIndex::ElasticScalingMVP as usize), + ), + scheduler_params: SchedulerParams { + lookahead: 2, + group_rotation_frequency: 20, + paras_availability_period: 4, + ..Default::default() + }, ..Default::default() } } @@ -891,7 +901,10 @@ pub fn rococo_testnet_genesis( "sudo": { "key": Some(root_key.clone()) }, "configuration": { "config": polkadot_runtime_parachains::configuration::HostConfiguration { - max_validators_per_core: Some(1), + scheduler_params: SchedulerParams { + max_validators_per_core: Some(1), + ..default_parachains_host_configuration().scheduler_params + }, ..default_parachains_host_configuration() }, }, diff --git a/polkadot/node/service/src/fake_runtime_api.rs b/polkadot/node/service/src/fake_runtime_api.rs index ccc3da22400d..085ea93fdc78 100644 --- a/polkadot/node/service/src/fake_runtime_api.rs +++ b/polkadot/node/service/src/fake_runtime_api.rs @@ -60,7 +60,7 @@ sp_api::impl_runtime_apis! { unimplemented!() } - fn initialize_block(_: &::Header) { + fn initialize_block(_: &::Header) -> sp_runtime::ExtrinsicInclusionMode { unimplemented!() } } diff --git a/polkadot/node/test/service/src/chain_spec.rs b/polkadot/node/test/service/src/chain_spec.rs index 4cc387317c3f..f14fa9fde58b 100644 --- a/polkadot/node/test/service/src/chain_spec.rs +++ b/polkadot/node/test/service/src/chain_spec.rs @@ -19,7 +19,9 @@ use babe_primitives::AuthorityId as BabeId; use grandpa::AuthorityId as GrandpaId; use pallet_staking::Forcing; -use polkadot_primitives::{AccountId, AssignmentId, ValidatorId, MAX_CODE_SIZE, MAX_POV_SIZE}; +use polkadot_primitives::{ + vstaging::SchedulerParams, AccountId, AssignmentId, ValidatorId, MAX_CODE_SIZE, MAX_POV_SIZE, +}; use polkadot_service::chain_spec::{get_account_id_from_seed, get_from_seed, Extensions}; use polkadot_test_runtime::BABE_GENESIS_EPOCH_CONFIG; use sc_chain_spec::{ChainSpec, ChainType}; @@ -165,11 +167,14 @@ fn polkadot_testnet_genesis( max_code_size: MAX_CODE_SIZE, max_pov_size: MAX_POV_SIZE, max_head_data_size: 32 * 1024, - group_rotation_frequency: 20, - paras_availability_period: 4, no_show_slots: 10, minimum_validation_upgrade_delay: 5, max_downward_message_size: 1024, + scheduler_params: SchedulerParams { + group_rotation_frequency: 20, + paras_availability_period: 4, + ..Default::default() + }, ..Default::default() }, } diff --git a/polkadot/primitives/src/v6/executor_params.rs b/polkadot/primitives/src/v6/executor_params.rs index 112a529f62b0..1e19f3b23fec 100644 --- a/polkadot/primitives/src/v6/executor_params.rs +++ b/polkadot/primitives/src/v6/executor_params.rs @@ -159,7 +159,9 @@ impl sp_std::fmt::LowerHex for ExecutorParamsHash { // into individual fields of the structure. Thus, complex migrations shall be avoided when adding // new entries and removing old ones. At the moment, there's no mandatory parameters defined. If // they show up, they must be clearly documented as mandatory ones. -#[derive(Clone, Debug, Encode, Decode, PartialEq, Eq, TypeInfo, Serialize, Deserialize)] +#[derive( + Clone, Debug, Default, Encode, Decode, PartialEq, Eq, TypeInfo, Serialize, Deserialize, +)] pub struct ExecutorParams(Vec); impl ExecutorParams { @@ -334,9 +336,3 @@ impl From<&[ExecutorParam]> for ExecutorParams { ExecutorParams(arr.to_vec()) } } - -impl Default for ExecutorParams { - fn default() -> Self { - ExecutorParams(vec![]) - } -} diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 39d9dfc02c5b..bd2a5106c44d 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -23,6 +23,7 @@ use sp_std::prelude::*; use parity_scale_codec::{Decode, Encode}; use primitives::RuntimeDebug; use scale_info::TypeInfo; +use sp_arithmetic::Perbill; /// Approval voting configuration parameters #[derive( @@ -50,6 +51,81 @@ impl Default for ApprovalVotingParams { } } +/// Scheduler configuration parameters. All coretime/ondemand parameters are here. +#[derive( + RuntimeDebug, + Copy, + Clone, + PartialEq, + Encode, + Decode, + TypeInfo, + serde::Serialize, + serde::Deserialize, +)] +pub struct SchedulerParams { + /// How often parachain groups should be rotated across parachains. + /// + /// Must be non-zero. + pub group_rotation_frequency: BlockNumber, + /// Availability timeout for a block on a core, measured in blocks. + /// + /// This is the maximum amount of blocks after a core became occupied that validators have time + /// to make the block available. + /// + /// This value only has effect on group rotations. If backers backed something at the end of + /// their rotation, the occupied core affects the backing group that comes afterwards. We limit + /// the effect one backing group can have on the next to `paras_availability_period` blocks. + /// + /// Within a group rotation there is no timeout as backers are only affecting themselves. + /// + /// Must be at least 1. With a value of 1, the previous group will not be able to negatively + /// affect the following group at the expense of a tight availability timeline at group + /// rotation boundaries. + pub paras_availability_period: BlockNumber, + /// The maximum number of validators to have per core. + /// + /// `None` means no maximum. + pub max_validators_per_core: Option, + /// The amount of blocks ahead to schedule paras. + pub lookahead: u32, + /// How many cores are managed by the coretime chain. + pub num_cores: u32, + /// The max number of times a claim can time out in availability. + pub max_availability_timeouts: u32, + /// The maximum queue size of the pay as you go module. + pub on_demand_queue_max_size: u32, + /// The target utilization of the spot price queue in percentages. + pub on_demand_target_queue_utilization: Perbill, + /// How quickly the fee rises in reaction to increased utilization. + /// The lower the number the slower the increase. + pub on_demand_fee_variability: Perbill, + /// The minimum amount needed to claim a slot in the spot pricing queue. + pub on_demand_base_fee: Balance, + /// The number of blocks a claim stays in the scheduler's claimqueue before getting cleared. + /// This number should go reasonably higher than the number of blocks in the async backing + /// lookahead. + pub ttl: BlockNumber, +} + +impl> Default for SchedulerParams { + fn default() -> Self { + Self { + group_rotation_frequency: 1u32.into(), + paras_availability_period: 1u32.into(), + max_validators_per_core: Default::default(), + lookahead: 1, + num_cores: Default::default(), + max_availability_timeouts: Default::default(), + on_demand_queue_max_size: ON_DEMAND_DEFAULT_QUEUE_MAX_SIZE, + on_demand_target_queue_utilization: Perbill::from_percent(25), + on_demand_fee_variability: Perbill::from_percent(3), + on_demand_base_fee: 10_000_000u128, + ttl: 5u32.into(), + } + } +} + use bitvec::vec::BitVec; /// Bit indices in the `HostConfiguration.node_features` that correspond to different node features. diff --git a/polkadot/roadmap/implementers-guide/src/node/utility/pvf-host-and-workers.md b/polkadot/roadmap/implementers-guide/src/node/utility/pvf-host-and-workers.md index e0984bd58d1d..39a317a07c84 100644 --- a/polkadot/roadmap/implementers-guide/src/node/utility/pvf-host-and-workers.md +++ b/polkadot/roadmap/implementers-guide/src/node/utility/pvf-host-and-workers.md @@ -125,6 +125,14 @@ execution request: reason, which may or may not be independent of the candidate or PVF. 5. **Internal errors:** See "Internal Errors" section. In this case, after the retry we abstain from voting. +6. **RuntimeConstruction** error. The precheck handles a general case of a wrong + artifact but doesn't guarantee its consistency between the preparation and + the execution. If something happened with the artifact between + the preparation of the artifact and its execution (e.g. the artifact was + corrupted on disk or a dirty node upgrade happened when the prepare worker + has a wasmtime version different from the execute worker's wasmtime version). + We treat such an error as possibly transient due to local issues and retry + one time. ### Preparation timeouts diff --git a/polkadot/roadmap/implementers-guide/src/types/runtime.md b/polkadot/roadmap/implementers-guide/src/types/runtime.md index 4b97409f8df3..1dfabd96db7c 100644 --- a/polkadot/roadmap/implementers-guide/src/types/runtime.md +++ b/polkadot/roadmap/implementers-guide/src/types/runtime.md @@ -4,106 +4,24 @@ Types used within the runtime exclusively and pervasively. ## Host Configuration -The internal-to-runtime configuration of the parachain host. This is expected to be altered only by governance procedures. - -```rust -struct HostConfiguration { - /// The minimum period, in blocks, between which parachains can update their validation code. - pub validation_upgrade_cooldown: BlockNumber, - /// The delay, in blocks, before a validation upgrade is applied. - pub validation_upgrade_delay: BlockNumber, - /// How long to keep code on-chain, in blocks. This should be sufficiently long that disputes - /// have concluded. - pub code_retention_period: BlockNumber, - /// The maximum validation code size, in bytes. - pub max_code_size: u32, - /// The maximum head-data size, in bytes. - pub max_head_data_size: u32, - /// The amount of availability cores to dedicate to parathreads (on-demand parachains). - pub parathread_cores: u32, - /// The number of retries that a parathread (on-demand parachain) author has to submit their block. - pub parathread_retries: u32, - /// How often parachain groups should be rotated across parachains. - pub group_rotation_frequency: BlockNumber, - /// The availability period, in blocks, for parachains. This is the amount of blocks - /// after inclusion that validators have to make the block available and signal its availability to - /// the chain. Must be at least 1. - pub chain_availability_period: BlockNumber, - /// The availability period, in blocks, for parathreads (on-demand parachains). Same as the `chain_availability_period`, - /// but a differing timeout due to differing requirements. Must be at least 1. - pub thread_availability_period: BlockNumber, - /// The amount of blocks ahead to schedule on-demand parachains. - pub scheduling_lookahead: u32, - /// The maximum number of validators to have per core. `None` means no maximum. - pub max_validators_per_core: Option, - /// The maximum number of validators to use for parachains, in total. `None` means no maximum. - pub max_validators: Option, - /// The amount of sessions to keep for disputes. - pub dispute_period: SessionIndex, - /// How long after dispute conclusion to accept statements. - pub dispute_post_conclusion_acceptance_period: BlockNumber, - /// The maximum number of dispute spam slots - pub dispute_max_spam_slots: u32, - /// The amount of consensus slots that must pass between submitting an assignment and - /// submitting an approval vote before a validator is considered a no-show. - /// Must be at least 1. - pub no_show_slots: u32, - /// The number of delay tranches in total. - pub n_delay_tranches: u32, - /// The width of the zeroth delay tranche for approval assignments. This many delay tranches - /// beyond 0 are all consolidated to form a wide 0 tranche. - pub zeroth_delay_tranche_width: u32, - /// The number of validators needed to approve a block. - pub needed_approvals: u32, - /// The number of samples to use in `RelayVRFModulo` or `RelayVRFModuloCompact` approval assignment criterions. - pub relay_vrf_modulo_samples: u32, - /// Total number of individual messages allowed in the parachain -> relay-chain message queue. - pub max_upward_queue_count: u32, - /// Total size of messages allowed in the parachain -> relay-chain message queue before which - /// no further messages may be added to it. If it exceeds this then the queue may contain only - /// a single message. - pub max_upward_queue_size: u32, - /// The maximum size of an upward message that can be sent by a candidate. - /// - /// This parameter affects the upper bound of size of `CandidateCommitments`. - pub max_upward_message_size: u32, - /// The maximum number of messages that a candidate can contain. - /// - /// This parameter affects the upper bound of size of `CandidateCommitments`. - pub max_upward_message_num_per_candidate: u32, - /// The maximum size of a message that can be put in a downward message queue. - /// - /// Since we require receiving at least one DMP message the obvious upper bound of the size is - /// the PoV size. Of course, there is a lot of other different things that a parachain may - /// decide to do with its PoV so this value in practice will be picked as a fraction of the PoV - /// size. - pub max_downward_message_size: u32, - /// The deposit that the sender should provide for opening an HRMP channel. - pub hrmp_sender_deposit: u32, - /// The deposit that the recipient should provide for accepting opening an HRMP channel. - pub hrmp_recipient_deposit: u32, - /// The maximum number of messages allowed in an HRMP channel at once. - pub hrmp_channel_max_capacity: u32, - /// The maximum total size of messages in bytes allowed in an HRMP channel at once. - pub hrmp_channel_max_total_size: u32, - /// The maximum number of inbound HRMP channels a parachain is allowed to accept. - pub hrmp_max_parachain_inbound_channels: u32, - /// The maximum number of inbound HRMP channels a parathread (on-demand parachain) is allowed to accept. - pub hrmp_max_parathread_inbound_channels: u32, - /// The maximum size of a message that could ever be put into an HRMP channel. - /// - /// This parameter affects the upper bound of size of `CandidateCommitments`. - pub hrmp_channel_max_message_size: u32, - /// The maximum number of outbound HRMP channels a parachain is allowed to open. - pub hrmp_max_parachain_outbound_channels: u32, - /// The maximum number of outbound HRMP channels a parathread (on-demand parachain) is allowed to open. - pub hrmp_max_parathread_outbound_channels: u32, - /// The maximum number of outbound HRMP messages can be sent by a candidate. - /// - /// This parameter affects the upper bound of size of `CandidateCommitments`. - pub hrmp_max_message_num_per_candidate: u32, -} -``` +The internal-to-runtime configuration of the parachain host is kept in `struct HostConfiguration`. This is expected to +be altered only by governance procedures or via migrations from the Polkadot-SDK codebase. The latest definition of +`HostConfiguration` can be found in the project repo +[here](https://github.com/paritytech/polkadot-sdk/blob/master/polkadot/runtime/parachains/src/configuration.rs). Each +parameter has got a doc comment so for any details please refer to the code. + +Some related parameters in `HostConfiguration` are grouped together so that they can be managed easily. These are: +* `async_backing_params` in `struct AsyncBackingParams` +* `executor_params` in `struct ExecutorParams` +* `approval_voting_params` in `struct ApprovalVotingParams` +* `scheduler_params` in `struct SchedulerParams` + +Check the definitions of these structs for further details. + +### Configuration migrations +Modifying `HostConfiguration` requires a storage migration. These migrations are located in the +[`migrations`](https://github.com/paritytech/polkadot-sdk/blob/master/polkadot/runtime/parachains/src/configuration.rs) +subfolder of Polkadot-SDK repo. ## ParaInherentData diff --git a/polkadot/runtime/common/src/claims.rs b/polkadot/runtime/common/src/claims.rs index 86550ea8b4eb..68f42914e447 100644 --- a/polkadot/runtime/common/src/claims.rs +++ b/polkadot/runtime/common/src/claims.rs @@ -702,7 +702,6 @@ mod tests { use secp_utils::*; use parity_scale_codec::Encode; - use sp_core::H256; // The testing primitives are very useful for avoiding having to work with signatures // or public keys. `u64` is used as the `AccountId` and no `Signature`s are required. use crate::claims; @@ -715,11 +714,8 @@ mod tests { }; use pallet_balances; use sp_runtime::{ - traits::{BlakeTwo256, Identity, IdentityLookup}, - transaction_validity::TransactionLongevity, - BuildStorage, - DispatchError::BadOrigin, - TokenError, + traits::Identity, transaction_validity::TransactionLongevity, BuildStorage, + DispatchError::BadOrigin, TokenError, }; type Block = frame_system::mocking::MockBlock; @@ -734,34 +730,13 @@ mod tests { } ); - parameter_types! { - pub const BlockHashCount: u32 = 250; - } - #[derive_impl(frame_system::config_preludes::TestDefaultConfig as frame_system::DefaultConfig)] impl frame_system::Config for Test { - type BaseCallFilter = frame_support::traits::Everything; - type BlockWeights = (); - type BlockLength = (); - type DbWeight = (); type RuntimeOrigin = RuntimeOrigin; type RuntimeCall = RuntimeCall; - type Nonce = u64; - type Hash = H256; - type Hashing = BlakeTwo256; - type AccountId = u64; - type Lookup = IdentityLookup; type Block = Block; type RuntimeEvent = RuntimeEvent; - type BlockHashCount = BlockHashCount; - type Version = (); - type PalletInfo = PalletInfo; type AccountData = pallet_balances::AccountData; - type OnNewAccount = (); - type OnKilledAccount = (); - type SystemWeightInfo = (); - type SS58Prefix = (); - type OnSetCode = (); type MaxConsumers = frame_support::traits::ConstU32<16>; } diff --git a/polkadot/runtime/parachains/src/assigner_coretime/mock_helpers.rs b/polkadot/runtime/parachains/src/assigner_coretime/mock_helpers.rs index 71c3f1fa39f7..e2ba0b4f7ea5 100644 --- a/polkadot/runtime/parachains/src/assigner_coretime/mock_helpers.rs +++ b/polkadot/runtime/parachains/src/assigner_coretime/mock_helpers.rs @@ -64,11 +64,12 @@ impl GenesisConfigBuilder { pub(super) fn build(self) -> MockGenesisConfig { let mut genesis = default_genesis_config(); let config = &mut genesis.configuration.config; - config.coretime_cores = self.on_demand_cores; - config.on_demand_base_fee = self.on_demand_base_fee; - config.on_demand_fee_variability = self.on_demand_fee_variability; - config.on_demand_queue_max_size = self.on_demand_max_queue_size; - config.on_demand_target_queue_utilization = self.on_demand_target_queue_utilization; + config.scheduler_params.num_cores = self.on_demand_cores; + config.scheduler_params.on_demand_base_fee = self.on_demand_base_fee; + config.scheduler_params.on_demand_fee_variability = self.on_demand_fee_variability; + config.scheduler_params.on_demand_queue_max_size = self.on_demand_max_queue_size; + config.scheduler_params.on_demand_target_queue_utilization = + self.on_demand_target_queue_utilization; let paras = &mut genesis.paras.paras; for para_id in self.onboarded_on_demand_chains { diff --git a/polkadot/runtime/parachains/src/assigner_coretime/mod.rs b/polkadot/runtime/parachains/src/assigner_coretime/mod.rs index 9da81dc816ca..e2da89fadd48 100644 --- a/polkadot/runtime/parachains/src/assigner_coretime/mod.rs +++ b/polkadot/runtime/parachains/src/assigner_coretime/mod.rs @@ -30,7 +30,7 @@ mod tests; use crate::{ assigner_on_demand, configuration, paras::AssignCoretime, - scheduler::common::{Assignment, AssignmentProvider, AssignmentProviderConfig}, + scheduler::common::{Assignment, AssignmentProvider}, ParaId, }; @@ -316,14 +316,6 @@ impl AssignmentProvider> for Pallet { } } - fn get_provider_config(_core_idx: CoreIndex) -> AssignmentProviderConfig> { - let config = >::config(); - AssignmentProviderConfig { - max_availability_timeouts: config.on_demand_retries, - ttl: config.on_demand_ttl, - } - } - #[cfg(any(feature = "runtime-benchmarks", test))] fn get_mock_assignment(_: CoreIndex, para_id: primitives::Id) -> Assignment { // Given that we are not tracking anything in `Bulk` assignments, it is safe to always @@ -333,7 +325,7 @@ impl AssignmentProvider> for Pallet { fn session_core_count() -> u32 { let config = >::config(); - config.coretime_cores + config.scheduler_params.num_cores } } @@ -482,8 +474,8 @@ impl AssignCoretime for Pallet { // Add a new core and assign the para to it. let mut config = >::config(); - let core = config.coretime_cores; - config.coretime_cores.saturating_inc(); + let core = config.scheduler_params.num_cores; + config.scheduler_params.num_cores.saturating_inc(); // `assign_coretime` is only called at genesis or by root, so setting the active // config here is fine. diff --git a/polkadot/runtime/parachains/src/assigner_on_demand/benchmarking.rs b/polkadot/runtime/parachains/src/assigner_on_demand/benchmarking.rs index 5a6060cd2b4e..8360e7a78d0a 100644 --- a/polkadot/runtime/parachains/src/assigner_on_demand/benchmarking.rs +++ b/polkadot/runtime/parachains/src/assigner_on_demand/benchmarking.rs @@ -43,7 +43,7 @@ where { ParasShared::::set_session_index(SESSION_INDEX); let mut config = HostConfiguration::default(); - config.coretime_cores = 1; + config.scheduler_params.num_cores = 1; ConfigurationPallet::::force_set_active_config(config); let mut parachains = ParachainsCache::new(); ParasPallet::::initialize_para_now( diff --git a/polkadot/runtime/parachains/src/assigner_on_demand/mock_helpers.rs b/polkadot/runtime/parachains/src/assigner_on_demand/mock_helpers.rs index de30330ac84e..f8d1a894f0e4 100644 --- a/polkadot/runtime/parachains/src/assigner_on_demand/mock_helpers.rs +++ b/polkadot/runtime/parachains/src/assigner_on_demand/mock_helpers.rs @@ -63,11 +63,12 @@ impl GenesisConfigBuilder { pub(super) fn build(self) -> MockGenesisConfig { let mut genesis = default_genesis_config(); let config = &mut genesis.configuration.config; - config.coretime_cores = self.on_demand_cores; - config.on_demand_base_fee = self.on_demand_base_fee; - config.on_demand_fee_variability = self.on_demand_fee_variability; - config.on_demand_queue_max_size = self.on_demand_max_queue_size; - config.on_demand_target_queue_utilization = self.on_demand_target_queue_utilization; + config.scheduler_params.num_cores = self.on_demand_cores; + config.scheduler_params.on_demand_base_fee = self.on_demand_base_fee; + config.scheduler_params.on_demand_fee_variability = self.on_demand_fee_variability; + config.scheduler_params.on_demand_queue_max_size = self.on_demand_max_queue_size; + config.scheduler_params.on_demand_target_queue_utilization = + self.on_demand_target_queue_utilization; let paras = &mut genesis.paras.paras; for para_id in self.onboarded_on_demand_chains { diff --git a/polkadot/runtime/parachains/src/assigner_on_demand/mod.rs b/polkadot/runtime/parachains/src/assigner_on_demand/mod.rs index 1b746e88694c..bc450dc78129 100644 --- a/polkadot/runtime/parachains/src/assigner_on_demand/mod.rs +++ b/polkadot/runtime/parachains/src/assigner_on_demand/mod.rs @@ -201,10 +201,10 @@ pub mod pallet { let old_traffic = SpotTraffic::::get(); match Self::calculate_spot_traffic( old_traffic, - config.on_demand_queue_max_size, + config.scheduler_params.on_demand_queue_max_size, Self::queue_size(), - config.on_demand_target_queue_utilization, - config.on_demand_fee_variability, + config.scheduler_params.on_demand_target_queue_utilization, + config.scheduler_params.on_demand_fee_variability, ) { Ok(new_traffic) => { // Only update storage on change @@ -330,8 +330,9 @@ where let traffic = SpotTraffic::::get(); // Calculate spot price - let spot_price: BalanceOf = - traffic.saturating_mul_int(config.on_demand_base_fee.saturated_into::>()); + let spot_price: BalanceOf = traffic.saturating_mul_int( + config.scheduler_params.on_demand_base_fee.saturated_into::>(), + ); // Is the current price higher than `max_amount` ensure!(spot_price.le(&max_amount), Error::::SpotPriceHigherThanMaxAmount); @@ -450,7 +451,10 @@ where OnDemandQueue::::try_mutate(|queue| { // Abort transaction if queue is too large - ensure!(Self::queue_size() < config.on_demand_queue_max_size, Error::::QueueFull); + ensure!( + Self::queue_size() < config.scheduler_params.on_demand_queue_max_size, + Error::::QueueFull + ); match location { QueuePushDirection::Back => queue.push_back(order), QueuePushDirection::Front => queue.push_front(order), @@ -472,7 +476,7 @@ where target: LOG_TARGET, "Failed to fetch the on demand queue size, returning the max size." ); - return config.on_demand_queue_max_size + return config.scheduler_params.on_demand_queue_max_size }, } } diff --git a/polkadot/runtime/parachains/src/assigner_parachains.rs b/polkadot/runtime/parachains/src/assigner_parachains.rs index 34b5d3c1ec51..b5f342563e97 100644 --- a/polkadot/runtime/parachains/src/assigner_parachains.rs +++ b/polkadot/runtime/parachains/src/assigner_parachains.rs @@ -27,7 +27,7 @@ use primitives::CoreIndex; use crate::{ configuration, paras, - scheduler::common::{Assignment, AssignmentProvider, AssignmentProviderConfig}, + scheduler::common::{Assignment, AssignmentProvider}, }; pub use pallet::*; @@ -58,16 +58,6 @@ impl AssignmentProvider> for Pallet { /// this is a no-op in the case of a bulk assignment slot. fn push_back_assignment(_: Assignment) {} - fn get_provider_config(_core_idx: CoreIndex) -> AssignmentProviderConfig> { - AssignmentProviderConfig { - // The next assignment already goes to the same [`ParaId`], no timeout tracking needed. - max_availability_timeouts: 0, - // The next assignment already goes to the same [`ParaId`], this can be any number - // that's high enough to clear the time it takes to clear backing/availability. - ttl: 10u32.into(), - } - } - #[cfg(any(feature = "runtime-benchmarks", test))] fn get_mock_assignment(_: CoreIndex, para_id: primitives::Id) -> Assignment { Assignment::Bulk(para_id) diff --git a/polkadot/runtime/parachains/src/assigner_parachains/mock_helpers.rs b/polkadot/runtime/parachains/src/assigner_parachains/mock_helpers.rs index e6e9fb074aa9..a46e114daeaf 100644 --- a/polkadot/runtime/parachains/src/assigner_parachains/mock_helpers.rs +++ b/polkadot/runtime/parachains/src/assigner_parachains/mock_helpers.rs @@ -60,11 +60,12 @@ impl GenesisConfigBuilder { pub(super) fn build(self) -> MockGenesisConfig { let mut genesis = default_genesis_config(); let config = &mut genesis.configuration.config; - config.coretime_cores = self.on_demand_cores; - config.on_demand_base_fee = self.on_demand_base_fee; - config.on_demand_fee_variability = self.on_demand_fee_variability; - config.on_demand_queue_max_size = self.on_demand_max_queue_size; - config.on_demand_target_queue_utilization = self.on_demand_target_queue_utilization; + config.scheduler_params.num_cores = self.on_demand_cores; + config.scheduler_params.on_demand_base_fee = self.on_demand_base_fee; + config.scheduler_params.on_demand_fee_variability = self.on_demand_fee_variability; + config.scheduler_params.on_demand_queue_max_size = self.on_demand_max_queue_size; + config.scheduler_params.on_demand_target_queue_utilization = + self.on_demand_target_queue_utilization; let paras = &mut genesis.paras.paras; for para_id in self.onboarded_on_demand_chains { diff --git a/polkadot/runtime/parachains/src/builder.rs b/polkadot/runtime/parachains/src/builder.rs index 500bc70cfa75..5b218addb1a2 100644 --- a/polkadot/runtime/parachains/src/builder.rs +++ b/polkadot/runtime/parachains/src/builder.rs @@ -18,23 +18,20 @@ use crate::{ configuration, inclusion, initializer, paras, paras::ParaKind, paras_inherent, - scheduler::{ - self, - common::{AssignmentProvider, AssignmentProviderConfig}, - CoreOccupied, ParasEntry, - }, + scheduler::{self, common::AssignmentProvider, CoreOccupied, ParasEntry}, session_info, shared, }; use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; use frame_support::pallet_prelude::*; use frame_system::pallet_prelude::*; use primitives::{ - collator_signature_payload, AvailabilityBitfield, BackedCandidate, CandidateCommitments, - CandidateDescriptor, CandidateHash, CollatorId, CollatorSignature, CommittedCandidateReceipt, - CompactStatement, CoreIndex, DisputeStatement, DisputeStatementSet, GroupIndex, HeadData, - Id as ParaId, IndexedVec, InherentData as ParachainsInherentData, InvalidDisputeStatementKind, - PersistedValidationData, SessionIndex, SigningContext, UncheckedSigned, - ValidDisputeStatementKind, ValidationCode, ValidatorId, ValidatorIndex, ValidityAttestation, + collator_signature_payload, vstaging::node_features::FeatureIndex, AvailabilityBitfield, + BackedCandidate, CandidateCommitments, CandidateDescriptor, CandidateHash, CollatorId, + CollatorSignature, CommittedCandidateReceipt, CompactStatement, CoreIndex, DisputeStatement, + DisputeStatementSet, GroupIndex, HeadData, Id as ParaId, IndexedVec, + InherentData as ParachainsInherentData, InvalidDisputeStatementKind, PersistedValidationData, + SessionIndex, SigningContext, UncheckedSigned, ValidDisputeStatementKind, ValidationCode, + ValidatorId, ValidatorIndex, ValidityAttestation, }; use sp_core::{sr25519, H256}; use sp_runtime::{ @@ -197,7 +194,10 @@ impl BenchBuilder { /// Maximum number of validators per core (a.k.a. max validators per group). This value is used /// if none is explicitly set on the builder. pub(crate) fn fallback_max_validators_per_core() -> u32 { - configuration::Pallet::::config().max_validators_per_core.unwrap_or(5) + configuration::Pallet::::config() + .scheduler_params + .max_validators_per_core + .unwrap_or(5) } /// Specify a mapping of core index/ para id to the number of dispute statements for the @@ -510,7 +510,7 @@ impl BenchBuilder { .iter() .map(|(seed, num_votes)| { assert!(*num_votes <= validators.len() as u32); - let (para_id, _core_idx, group_idx) = self.create_indexes(*seed); + let (para_id, core_idx, group_idx) = self.create_indexes(*seed); // This generates a pair and adds it to the keystore, returning just the public. let collator_public = CollatorId::generate_pair(None); @@ -587,11 +587,18 @@ impl BenchBuilder { }) .collect(); + // Check if the elastic scaling bit is set, if so we need to supply the core index + // in the generated candidate. + let core_idx = configuration::Pallet::::config() + .node_features + .get(FeatureIndex::ElasticScalingMVP as usize) + .map(|_the_bit| core_idx); + BackedCandidate::::new( candidate, validity_votes, bitvec::bitvec![u8, bitvec::order::Lsb0; 1; group_validators.len()], - None, + core_idx, ) }) .collect() @@ -684,7 +691,7 @@ impl BenchBuilder { // We are currently in Session 0, so these changes will take effect in Session 2. Self::setup_para_ids(used_cores); configuration::ActiveConfig::::mutate(|c| { - c.coretime_cores = used_cores; + c.scheduler_params.num_cores = used_cores; }); let validator_ids = Self::generate_validator_pairs(self.max_validators()); @@ -715,8 +722,7 @@ impl BenchBuilder { let cores = (0..used_cores) .into_iter() .map(|i| { - let AssignmentProviderConfig { ttl, .. } = - scheduler::Pallet::::assignment_provider_config(CoreIndex(i)); + let ttl = configuration::Pallet::::config().scheduler_params.ttl; // Load an assignment into provider so that one is present to pop let assignment = ::AssignmentProvider::get_mock_assignment( CoreIndex(i), @@ -731,8 +737,7 @@ impl BenchBuilder { let cores = (0..used_cores) .into_iter() .map(|i| { - let AssignmentProviderConfig { ttl, .. } = - scheduler::Pallet::::assignment_provider_config(CoreIndex(i)); + let ttl = configuration::Pallet::::config().scheduler_params.ttl; // Load an assignment into provider so that one is present to pop let assignment = ::AssignmentProvider::get_mock_assignment( diff --git a/polkadot/runtime/parachains/src/configuration.rs b/polkadot/runtime/parachains/src/configuration.rs index b0e9d03df886..364a15215d38 100644 --- a/polkadot/runtime/parachains/src/configuration.rs +++ b/polkadot/runtime/parachains/src/configuration.rs @@ -29,7 +29,6 @@ use primitives::{ vstaging::{ApprovalVotingParams, NodeFeatures}, AsyncBackingParams, Balance, ExecutorParamError, ExecutorParams, SessionIndex, LEGACY_MIN_BACKING_VOTES, MAX_CODE_SIZE, MAX_HEAD_DATA_SIZE, MAX_POV_SIZE, - ON_DEMAND_DEFAULT_QUEUE_MAX_SIZE, }; use sp_runtime::{traits::Zero, Perbill}; use sp_std::prelude::*; @@ -43,6 +42,7 @@ mod benchmarking; pub mod migration; pub use pallet::*; +use primitives::vstaging::SchedulerParams; const LOG_TARGET: &str = "runtime::configuration"; @@ -118,9 +118,9 @@ pub struct HostConfiguration { /// been completed. /// /// Note, there are situations in which `expected_at` in the past. For example, if - /// [`paras_availability_period`](Self::paras_availability_period) is less than the delay set - /// by this field or if PVF pre-check took more time than the delay. In such cases, the upgrade - /// is further at the earliest possible time determined by + /// [`paras_availability_period`](SchedulerParams::paras_availability_period) is less than the + /// delay set by this field or if PVF pre-check took more time than the delay. In such cases, + /// the upgrade is further at the earliest possible time determined by /// [`minimum_validation_upgrade_delay`](Self::minimum_validation_upgrade_delay). /// /// The rationale for this delay has to do with relay-chain reversions. In case there is an @@ -172,48 +172,7 @@ pub struct HostConfiguration { /// How long to keep code on-chain, in blocks. This should be sufficiently long that disputes /// have concluded. pub code_retention_period: BlockNumber, - /// How many cores are managed by the coretime chain. - pub coretime_cores: u32, - /// The number of retries that a on demand author has to submit their block. - pub on_demand_retries: u32, - /// The maximum queue size of the pay as you go module. - pub on_demand_queue_max_size: u32, - /// The target utilization of the spot price queue in percentages. - pub on_demand_target_queue_utilization: Perbill, - /// How quickly the fee rises in reaction to increased utilization. - /// The lower the number the slower the increase. - pub on_demand_fee_variability: Perbill, - /// The minimum amount needed to claim a slot in the spot pricing queue. - pub on_demand_base_fee: Balance, - /// The number of blocks an on demand claim stays in the scheduler's claimqueue before getting - /// cleared. This number should go reasonably higher than the number of blocks in the async - /// backing lookahead. - pub on_demand_ttl: BlockNumber, - /// How often parachain groups should be rotated across parachains. - /// - /// Must be non-zero. - pub group_rotation_frequency: BlockNumber, - /// The minimum availability period, in blocks. - /// - /// This is the minimum amount of blocks after a core became occupied that validators have time - /// to make the block available. - /// - /// This value only has effect on group rotations. If backers backed something at the end of - /// their rotation, the occupied core affects the backing group that comes afterwards. We limit - /// the effect one backing group can have on the next to `paras_availability_period` blocks. - /// - /// Within a group rotation there is no timeout as backers are only affecting themselves. - /// - /// Must be at least 1. With a value of 1, the previous group will not be able to negatively - /// affect the following group at the expense of a tight availability timeline at group - /// rotation boundaries. - pub paras_availability_period: BlockNumber, - /// The amount of blocks ahead to schedule paras. - pub scheduling_lookahead: u32, - /// The maximum number of validators to have per core. - /// - /// `None` means no maximum. - pub max_validators_per_core: Option, + /// The maximum number of validators to use for parachain consensus, period. /// /// `None` means no maximum. @@ -257,7 +216,7 @@ pub struct HostConfiguration { /// scheduled. This number is controlled by this field. /// /// This value should be greater than - /// [`paras_availability_period`](Self::paras_availability_period). + /// [`paras_availability_period`](SchedulerParams::paras_availability_period). pub minimum_validation_upgrade_delay: BlockNumber, /// The minimum number of valid backing statements required to consider a parachain candidate /// backable. @@ -266,6 +225,8 @@ pub struct HostConfiguration { pub node_features: NodeFeatures, /// Params used by approval-voting pub approval_voting_params: ApprovalVotingParams, + /// Scheduler parameters + pub scheduler_params: SchedulerParams, } impl> Default for HostConfiguration { @@ -275,8 +236,6 @@ impl> Default for HostConfiguration> Default for HostConfiguration> Default for HostConfiguration { ZeroMinimumBackingVotes, /// `executor_params` are inconsistent. InconsistentExecutorParams { inner: ExecutorParamError }, + /// TTL should be bigger than lookahead + LookaheadExceedsTTL, } impl HostConfiguration @@ -373,11 +326,11 @@ where pub fn check_consistency(&self) -> Result<(), InconsistentError> { use InconsistentError::*; - if self.group_rotation_frequency.is_zero() { + if self.scheduler_params.group_rotation_frequency.is_zero() { return Err(ZeroGroupRotationFrequency) } - if self.paras_availability_period.is_zero() { + if self.scheduler_params.paras_availability_period.is_zero() { return Err(ZeroParasAvailabilityPeriod) } @@ -399,10 +352,11 @@ where return Err(MaxPovSizeExceedHardLimit { max_pov_size: self.max_pov_size }) } - if self.minimum_validation_upgrade_delay <= self.paras_availability_period { + if self.minimum_validation_upgrade_delay <= self.scheduler_params.paras_availability_period + { return Err(MinimumValidationUpgradeDelayLessThanChainAvailabilityPeriod { minimum_validation_upgrade_delay: self.minimum_validation_upgrade_delay.clone(), - paras_availability_period: self.paras_availability_period.clone(), + paras_availability_period: self.scheduler_params.paras_availability_period.clone(), }) } @@ -447,6 +401,10 @@ where return Err(InconsistentExecutorParams { inner }) } + if self.scheduler_params.ttl < self.scheduler_params.lookahead.into() { + return Err(LookaheadExceedsTTL) + } + Ok(()) } @@ -471,6 +429,7 @@ pub trait WeightInfo { fn set_config_with_executor_params() -> Weight; fn set_config_with_perbill() -> Weight; fn set_node_feature() -> Weight; + fn set_config_with_scheduler_params() -> Weight; } pub struct TestWeightInfo; @@ -499,6 +458,9 @@ impl WeightInfo for TestWeightInfo { fn set_node_feature() -> Weight { Weight::MAX } + fn set_config_with_scheduler_params() -> Weight { + Weight::MAX + } } #[frame_support::pallet] @@ -520,7 +482,8 @@ pub mod pallet { /// v8-v9: /// v9-v10: /// v10-11: - const STORAGE_VERSION: StorageVersion = StorageVersion::new(11); + /// v11-12: + const STORAGE_VERSION: StorageVersion = StorageVersion::new(12); #[pallet::pallet] #[pallet::storage_version(STORAGE_VERSION)] @@ -679,16 +642,16 @@ pub mod pallet { Self::set_coretime_cores_unchecked(new) } - /// Set the number of retries for a particular on demand. + /// Set the max number of times a claim may timeout on a core before it is abandoned #[pallet::call_index(7)] #[pallet::weight(( T::WeightInfo::set_config_with_u32(), DispatchClass::Operational, ))] - pub fn set_on_demand_retries(origin: OriginFor, new: u32) -> DispatchResult { + pub fn set_max_availability_timeouts(origin: OriginFor, new: u32) -> DispatchResult { ensure_root(origin)?; Self::schedule_config_update(|config| { - config.on_demand_retries = new; + config.scheduler_params.max_availability_timeouts = new; }) } @@ -704,7 +667,7 @@ pub mod pallet { ) -> DispatchResult { ensure_root(origin)?; Self::schedule_config_update(|config| { - config.group_rotation_frequency = new; + config.scheduler_params.group_rotation_frequency = new; }) } @@ -720,7 +683,7 @@ pub mod pallet { ) -> DispatchResult { ensure_root(origin)?; Self::schedule_config_update(|config| { - config.paras_availability_period = new; + config.scheduler_params.paras_availability_period = new; }) } @@ -733,7 +696,7 @@ pub mod pallet { pub fn set_scheduling_lookahead(origin: OriginFor, new: u32) -> DispatchResult { ensure_root(origin)?; Self::schedule_config_update(|config| { - config.scheduling_lookahead = new; + config.scheduler_params.lookahead = new; }) } @@ -749,7 +712,7 @@ pub mod pallet { ) -> DispatchResult { ensure_root(origin)?; Self::schedule_config_update(|config| { - config.max_validators_per_core = new; + config.scheduler_params.max_validators_per_core = new; }) } @@ -1141,7 +1104,7 @@ pub mod pallet { pub fn set_on_demand_base_fee(origin: OriginFor, new: Balance) -> DispatchResult { ensure_root(origin)?; Self::schedule_config_update(|config| { - config.on_demand_base_fee = new; + config.scheduler_params.on_demand_base_fee = new; }) } @@ -1154,7 +1117,7 @@ pub mod pallet { pub fn set_on_demand_fee_variability(origin: OriginFor, new: Perbill) -> DispatchResult { ensure_root(origin)?; Self::schedule_config_update(|config| { - config.on_demand_fee_variability = new; + config.scheduler_params.on_demand_fee_variability = new; }) } @@ -1167,7 +1130,7 @@ pub mod pallet { pub fn set_on_demand_queue_max_size(origin: OriginFor, new: u32) -> DispatchResult { ensure_root(origin)?; Self::schedule_config_update(|config| { - config.on_demand_queue_max_size = new; + config.scheduler_params.on_demand_queue_max_size = new; }) } /// Set the on demand (parathreads) fee variability. @@ -1182,7 +1145,7 @@ pub mod pallet { ) -> DispatchResult { ensure_root(origin)?; Self::schedule_config_update(|config| { - config.on_demand_target_queue_utilization = new; + config.scheduler_params.on_demand_target_queue_utilization = new; }) } /// Set the on demand (parathreads) ttl in the claimqueue. @@ -1194,7 +1157,7 @@ pub mod pallet { pub fn set_on_demand_ttl(origin: OriginFor, new: BlockNumberFor) -> DispatchResult { ensure_root(origin)?; Self::schedule_config_update(|config| { - config.on_demand_ttl = new; + config.scheduler_params.ttl = new; }) } @@ -1244,6 +1207,22 @@ pub mod pallet { config.approval_voting_params = new; }) } + + /// Set scheduler-params. + #[pallet::call_index(55)] + #[pallet::weight(( + T::WeightInfo::set_config_with_scheduler_params(), + DispatchClass::Operational, + ))] + pub fn set_scheduler_params( + origin: OriginFor, + new: SchedulerParams>, + ) -> DispatchResult { + ensure_root(origin)?; + Self::schedule_config_update(|config| { + config.scheduler_params = new; + }) + } } impl Pallet { @@ -1252,7 +1231,7 @@ pub mod pallet { /// To be used if authorization is checked otherwise. pub fn set_coretime_cores_unchecked(new: u32) -> DispatchResult { Self::schedule_config_update(|config| { - config.coretime_cores = new; + config.scheduler_params.num_cores = new; }) } } @@ -1393,7 +1372,7 @@ impl Pallet { let base_config_consistent = base_config.check_consistency().is_ok(); // Now, we need to decide what the new configuration should be. - // We also move the `base_config` to `new_config` to empahsize that the base config was + // We also move the `base_config` to `new_config` to emphasize that the base config was // destroyed by the `updater`. updater(&mut base_config); let new_config = base_config; diff --git a/polkadot/runtime/parachains/src/configuration/benchmarking.rs b/polkadot/runtime/parachains/src/configuration/benchmarking.rs index 67daf1c45988..882b5aab096a 100644 --- a/polkadot/runtime/parachains/src/configuration/benchmarking.rs +++ b/polkadot/runtime/parachains/src/configuration/benchmarking.rs @@ -51,6 +51,8 @@ benchmarks! { set_node_feature{}: set_node_feature(RawOrigin::Root, 255, true) + set_config_with_scheduler_params {} : set_scheduler_params(RawOrigin::Root, SchedulerParams::default()) + impl_benchmark_test_suite!( Pallet, crate::mock::new_test_ext(Default::default()), diff --git a/polkadot/runtime/parachains/src/configuration/migration.rs b/polkadot/runtime/parachains/src/configuration/migration.rs index 2838b73092db..87b30b177e73 100644 --- a/polkadot/runtime/parachains/src/configuration/migration.rs +++ b/polkadot/runtime/parachains/src/configuration/migration.rs @@ -18,6 +18,7 @@ pub mod v10; pub mod v11; +pub mod v12; pub mod v6; pub mod v7; pub mod v8; diff --git a/polkadot/runtime/parachains/src/configuration/migration/v11.rs b/polkadot/runtime/parachains/src/configuration/migration/v11.rs index a69061ac1381..f6e0e0431640 100644 --- a/polkadot/runtime/parachains/src/configuration/migration/v11.rs +++ b/polkadot/runtime/parachains/src/configuration/migration/v11.rs @@ -21,13 +21,122 @@ use frame_support::{ migrations::VersionedMigration, pallet_prelude::*, traits::Defensive, weights::Weight, }; use frame_system::pallet_prelude::BlockNumberFor; -use primitives::{vstaging::ApprovalVotingParams, SessionIndex}; +use primitives::{ + vstaging::ApprovalVotingParams, AsyncBackingParams, ExecutorParams, SessionIndex, + LEGACY_MIN_BACKING_VOTES, ON_DEMAND_DEFAULT_QUEUE_MAX_SIZE, +}; use sp_std::vec::Vec; use frame_support::traits::OnRuntimeUpgrade; +use polkadot_core_primitives::Balance; +use primitives::vstaging::NodeFeatures; +use sp_arithmetic::Perbill; use super::v10::V10HostConfiguration; -type V11HostConfiguration = configuration::HostConfiguration; + +#[derive(Clone, Encode, PartialEq, Decode, Debug)] +pub struct V11HostConfiguration { + pub max_code_size: u32, + pub max_head_data_size: u32, + pub max_upward_queue_count: u32, + pub max_upward_queue_size: u32, + pub max_upward_message_size: u32, + pub max_upward_message_num_per_candidate: u32, + pub hrmp_max_message_num_per_candidate: u32, + pub validation_upgrade_cooldown: BlockNumber, + pub validation_upgrade_delay: BlockNumber, + pub async_backing_params: AsyncBackingParams, + pub max_pov_size: u32, + pub max_downward_message_size: u32, + pub hrmp_max_parachain_outbound_channels: u32, + pub hrmp_sender_deposit: Balance, + pub hrmp_recipient_deposit: Balance, + pub hrmp_channel_max_capacity: u32, + pub hrmp_channel_max_total_size: u32, + pub hrmp_max_parachain_inbound_channels: u32, + pub hrmp_channel_max_message_size: u32, + pub executor_params: ExecutorParams, + pub code_retention_period: BlockNumber, + pub coretime_cores: u32, + pub on_demand_retries: u32, + pub on_demand_queue_max_size: u32, + pub on_demand_target_queue_utilization: Perbill, + pub on_demand_fee_variability: Perbill, + pub on_demand_base_fee: Balance, + pub on_demand_ttl: BlockNumber, + pub group_rotation_frequency: BlockNumber, + pub paras_availability_period: BlockNumber, + pub scheduling_lookahead: u32, + pub max_validators_per_core: Option, + pub max_validators: Option, + pub dispute_period: SessionIndex, + pub dispute_post_conclusion_acceptance_period: BlockNumber, + pub no_show_slots: u32, + pub n_delay_tranches: u32, + pub zeroth_delay_tranche_width: u32, + pub needed_approvals: u32, + pub relay_vrf_modulo_samples: u32, + pub pvf_voting_ttl: SessionIndex, + pub minimum_validation_upgrade_delay: BlockNumber, + pub minimum_backing_votes: u32, + pub node_features: NodeFeatures, + pub approval_voting_params: ApprovalVotingParams, +} + +impl> Default for V11HostConfiguration { + fn default() -> Self { + Self { + async_backing_params: AsyncBackingParams { + max_candidate_depth: 0, + allowed_ancestry_len: 0, + }, + group_rotation_frequency: 1u32.into(), + paras_availability_period: 1u32.into(), + no_show_slots: 1u32.into(), + validation_upgrade_cooldown: Default::default(), + validation_upgrade_delay: 2u32.into(), + code_retention_period: Default::default(), + max_code_size: Default::default(), + max_pov_size: Default::default(), + max_head_data_size: Default::default(), + coretime_cores: Default::default(), + on_demand_retries: Default::default(), + scheduling_lookahead: 1, + max_validators_per_core: Default::default(), + max_validators: None, + dispute_period: 6, + dispute_post_conclusion_acceptance_period: 100.into(), + n_delay_tranches: Default::default(), + zeroth_delay_tranche_width: Default::default(), + needed_approvals: Default::default(), + relay_vrf_modulo_samples: Default::default(), + max_upward_queue_count: Default::default(), + max_upward_queue_size: Default::default(), + max_downward_message_size: Default::default(), + max_upward_message_size: Default::default(), + max_upward_message_num_per_candidate: Default::default(), + hrmp_sender_deposit: Default::default(), + hrmp_recipient_deposit: Default::default(), + hrmp_channel_max_capacity: Default::default(), + hrmp_channel_max_total_size: Default::default(), + hrmp_max_parachain_inbound_channels: Default::default(), + hrmp_channel_max_message_size: Default::default(), + hrmp_max_parachain_outbound_channels: Default::default(), + hrmp_max_message_num_per_candidate: Default::default(), + pvf_voting_ttl: 2u32.into(), + minimum_validation_upgrade_delay: 2.into(), + executor_params: Default::default(), + approval_voting_params: ApprovalVotingParams { max_approval_coalesce_count: 1 }, + on_demand_queue_max_size: ON_DEMAND_DEFAULT_QUEUE_MAX_SIZE, + on_demand_base_fee: 10_000_000u128, + on_demand_fee_variability: Perbill::from_percent(3), + on_demand_target_queue_utilization: Perbill::from_percent(25), + on_demand_ttl: 5u32.into(), + minimum_backing_votes: LEGACY_MIN_BACKING_VOTES, + node_features: NodeFeatures::EMPTY, + } + } +} mod v10 { use super::*; diff --git a/polkadot/runtime/parachains/src/configuration/migration/v12.rs b/polkadot/runtime/parachains/src/configuration/migration/v12.rs new file mode 100644 index 000000000000..4295a79893e8 --- /dev/null +++ b/polkadot/runtime/parachains/src/configuration/migration/v12.rs @@ -0,0 +1,349 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! A module that is responsible for migration of storage. + +use crate::configuration::{self, migration::v11::V11HostConfiguration, Config, Pallet}; +use frame_support::{ + migrations::VersionedMigration, + pallet_prelude::*, + traits::{Defensive, OnRuntimeUpgrade}, +}; +use frame_system::pallet_prelude::BlockNumberFor; +use primitives::vstaging::SchedulerParams; +use sp_core::Get; +use sp_staking::SessionIndex; +use sp_std::vec::Vec; + +type V12HostConfiguration = configuration::HostConfiguration; + +mod v11 { + use super::*; + + #[frame_support::storage_alias] + pub(crate) type ActiveConfig = + StorageValue, V11HostConfiguration>, OptionQuery>; + + #[frame_support::storage_alias] + pub(crate) type PendingConfigs = StorageValue< + Pallet, + Vec<(SessionIndex, V11HostConfiguration>)>, + OptionQuery, + >; +} + +mod v12 { + use super::*; + + #[frame_support::storage_alias] + pub(crate) type ActiveConfig = + StorageValue, V12HostConfiguration>, OptionQuery>; + + #[frame_support::storage_alias] + pub(crate) type PendingConfigs = StorageValue< + Pallet, + Vec<(SessionIndex, V12HostConfiguration>)>, + OptionQuery, + >; +} + +pub type MigrateToV12 = VersionedMigration< + 11, + 12, + UncheckedMigrateToV12, + Pallet, + ::DbWeight, +>; + +pub struct UncheckedMigrateToV12(sp_std::marker::PhantomData); + +impl OnRuntimeUpgrade for UncheckedMigrateToV12 { + #[cfg(feature = "try-runtime")] + fn pre_upgrade() -> Result, sp_runtime::TryRuntimeError> { + log::trace!(target: crate::configuration::LOG_TARGET, "Running pre_upgrade() for HostConfiguration MigrateToV12"); + Ok(Vec::new()) + } + + fn on_runtime_upgrade() -> Weight { + log::info!(target: configuration::LOG_TARGET, "HostConfiguration MigrateToV12 started"); + let weight_consumed = migrate_to_v12::(); + + log::info!(target: configuration::LOG_TARGET, "HostConfiguration MigrateToV12 executed successfully"); + + weight_consumed + } + + #[cfg(feature = "try-runtime")] + fn post_upgrade(_state: Vec) -> Result<(), sp_runtime::TryRuntimeError> { + log::trace!(target: crate::configuration::LOG_TARGET, "Running post_upgrade() for HostConfiguration MigrateToV12"); + ensure!( + StorageVersion::get::>() >= 12, + "Storage version should be >= 12 after the migration" + ); + + Ok(()) + } +} + +fn migrate_to_v12() -> Weight { + // Unusual formatting is justified: + // - make it easier to verify that fields assign what they supposed to assign. + // - this code is transient and will be removed after all migrations are done. + // - this code is important enough to optimize for legibility sacrificing consistency. + #[rustfmt::skip] + let translate = + |pre: V11HostConfiguration>| -> + V12HostConfiguration> + { + V12HostConfiguration { + max_code_size : pre.max_code_size, + max_head_data_size : pre.max_head_data_size, + max_upward_queue_count : pre.max_upward_queue_count, + max_upward_queue_size : pre.max_upward_queue_size, + max_upward_message_size : pre.max_upward_message_size, + max_upward_message_num_per_candidate : pre.max_upward_message_num_per_candidate, + hrmp_max_message_num_per_candidate : pre.hrmp_max_message_num_per_candidate, + validation_upgrade_cooldown : pre.validation_upgrade_cooldown, + validation_upgrade_delay : pre.validation_upgrade_delay, + max_pov_size : pre.max_pov_size, + max_downward_message_size : pre.max_downward_message_size, + hrmp_sender_deposit : pre.hrmp_sender_deposit, + hrmp_recipient_deposit : pre.hrmp_recipient_deposit, + hrmp_channel_max_capacity : pre.hrmp_channel_max_capacity, + hrmp_channel_max_total_size : pre.hrmp_channel_max_total_size, + hrmp_max_parachain_inbound_channels : pre.hrmp_max_parachain_inbound_channels, + hrmp_max_parachain_outbound_channels : pre.hrmp_max_parachain_outbound_channels, + hrmp_channel_max_message_size : pre.hrmp_channel_max_message_size, + code_retention_period : pre.code_retention_period, + max_validators : pre.max_validators, + dispute_period : pre.dispute_period, + dispute_post_conclusion_acceptance_period: pre.dispute_post_conclusion_acceptance_period, + no_show_slots : pre.no_show_slots, + n_delay_tranches : pre.n_delay_tranches, + zeroth_delay_tranche_width : pre.zeroth_delay_tranche_width, + needed_approvals : pre.needed_approvals, + relay_vrf_modulo_samples : pre.relay_vrf_modulo_samples, + pvf_voting_ttl : pre.pvf_voting_ttl, + minimum_validation_upgrade_delay : pre.minimum_validation_upgrade_delay, + async_backing_params : pre.async_backing_params, + executor_params : pre.executor_params, + minimum_backing_votes : pre.minimum_backing_votes, + node_features : pre.node_features, + approval_voting_params : pre.approval_voting_params, + scheduler_params: SchedulerParams { + group_rotation_frequency : pre.group_rotation_frequency, + paras_availability_period : pre.paras_availability_period, + max_validators_per_core : pre.max_validators_per_core, + lookahead : pre.scheduling_lookahead, + num_cores : pre.coretime_cores, + max_availability_timeouts : pre.on_demand_retries, + on_demand_queue_max_size : pre.on_demand_queue_max_size, + on_demand_target_queue_utilization : pre.on_demand_target_queue_utilization, + on_demand_fee_variability : pre.on_demand_fee_variability, + on_demand_base_fee : pre.on_demand_base_fee, + ttl : pre.on_demand_ttl, + } + } + }; + + let v11 = v11::ActiveConfig::::get() + .defensive_proof("Could not decode old config") + .unwrap_or_default(); + let v12 = translate(v11); + v12::ActiveConfig::::set(Some(v12)); + + // Allowed to be empty. + let pending_v11 = v11::PendingConfigs::::get().unwrap_or_default(); + let mut pending_v12 = Vec::new(); + + for (session, v11) in pending_v11.into_iter() { + let v12 = translate(v11); + pending_v12.push((session, v12)); + } + v12::PendingConfigs::::set(Some(pending_v12.clone())); + + let num_configs = (pending_v12.len() + 1) as u64; + T::DbWeight::get().reads_writes(num_configs, num_configs) +} + +#[cfg(test)] +mod tests { + use primitives::LEGACY_MIN_BACKING_VOTES; + use sp_arithmetic::Perbill; + + use super::*; + use crate::mock::{new_test_ext, Test}; + + #[test] + fn v12_deserialized_from_actual_data() { + // Example how to get new `raw_config`: + // We'll obtain the raw_config at a specified a block + // Steps: + // 1. Go to Polkadot.js -> Developer -> Chain state -> Storage: https://polkadot.js.org/apps/#/chainstate + // 2. Set these parameters: + // 2.1. selected state query: configuration; activeConfig(): + // PolkadotRuntimeParachainsConfigurationHostConfiguration + // 2.2. blockhash to query at: + // 0xf89d3ab5312c5f70d396dc59612f0aa65806c798346f9db4b35278baed2e0e53 (the hash of + // the block) + // 2.3. Note the value of encoded storage key -> + // 0x06de3d8a54d27e44a9d5ce189618f22db4b49d95320d9021994c850f25b8e385 for the + // referenced block. + // 2.4. You'll also need the decoded values to update the test. + // 3. Go to Polkadot.js -> Developer -> Chain state -> Raw storage + // 3.1 Enter the encoded storage key and you get the raw config. + + // This exceeds the maximal line width length, but that's fine, since this is not code and + // doesn't need to be read and also leaving it as one line allows to easily copy it. + let raw_config = + hex_literal::hex![ + "0000300000800000080000000000100000c8000005000000050000000200000002000000000000000000000000005000000010000400000000000000000000000000000000000000000000000000000000000000000000000800000000200000040000000000100000b004000000060000006400000002000000190000000000000002000000020000000200000005000000020000000001000000140000000400000001010000000100000001000000000000001027000080b2e60e80c3c9018096980000000000000000000000000005000000" + ]; + + let v12 = + V12HostConfiguration::::decode(&mut &raw_config[..]).unwrap(); + + // We check only a sample of the values here. If we missed any fields or messed up data + // types that would skew all the fields coming after. + assert_eq!(v12.max_code_size, 3_145_728); + assert_eq!(v12.validation_upgrade_cooldown, 2); + assert_eq!(v12.max_pov_size, 5_242_880); + assert_eq!(v12.hrmp_channel_max_message_size, 1_048_576); + assert_eq!(v12.n_delay_tranches, 25); + assert_eq!(v12.minimum_validation_upgrade_delay, 5); + assert_eq!(v12.minimum_backing_votes, LEGACY_MIN_BACKING_VOTES); + assert_eq!(v12.approval_voting_params.max_approval_coalesce_count, 1); + assert_eq!(v12.scheduler_params.group_rotation_frequency, 20); + assert_eq!(v12.scheduler_params.paras_availability_period, 4); + assert_eq!(v12.scheduler_params.lookahead, 1); + assert_eq!(v12.scheduler_params.num_cores, 1); + assert_eq!(v12.scheduler_params.max_availability_timeouts, 0); + assert_eq!(v12.scheduler_params.on_demand_queue_max_size, 10_000); + assert_eq!( + v12.scheduler_params.on_demand_target_queue_utilization, + Perbill::from_percent(25) + ); + assert_eq!(v12.scheduler_params.on_demand_fee_variability, Perbill::from_percent(3)); + assert_eq!(v12.scheduler_params.on_demand_base_fee, 10_000_000); + assert_eq!(v12.scheduler_params.ttl, 5); + } + + #[test] + fn test_migrate_to_v12() { + // Host configuration has lots of fields. However, in this migration we only add one + // field. The most important part to check are a couple of the last fields. We also pick + // extra fields to check arbitrarily, e.g. depending on their position (i.e. the middle) and + // also their type. + // + // We specify only the picked fields and the rest should be provided by the `Default` + // implementation. That implementation is copied over between the two types and should work + // fine. + let v11 = V11HostConfiguration:: { + needed_approvals: 69, + paras_availability_period: 55, + hrmp_recipient_deposit: 1337, + max_pov_size: 1111, + minimum_validation_upgrade_delay: 20, + on_demand_ttl: 3, + on_demand_retries: 10, + ..Default::default() + }; + + let mut pending_configs = Vec::new(); + pending_configs.push((100, v11.clone())); + pending_configs.push((300, v11.clone())); + + new_test_ext(Default::default()).execute_with(|| { + // Implant the v10 version in the state. + v11::ActiveConfig::::set(Some(v11.clone())); + v11::PendingConfigs::::set(Some(pending_configs)); + + migrate_to_v12::(); + + let v12 = v12::ActiveConfig::::get().unwrap(); + assert_eq!(v12.approval_voting_params.max_approval_coalesce_count, 1); + + let mut configs_to_check = v12::PendingConfigs::::get().unwrap(); + configs_to_check.push((0, v12.clone())); + + for (_, v12) in configs_to_check { + #[rustfmt::skip] + { + assert_eq!(v11.max_code_size , v12.max_code_size); + assert_eq!(v11.max_head_data_size , v12.max_head_data_size); + assert_eq!(v11.max_upward_queue_count , v12.max_upward_queue_count); + assert_eq!(v11.max_upward_queue_size , v12.max_upward_queue_size); + assert_eq!(v11.max_upward_message_size , v12.max_upward_message_size); + assert_eq!(v11.max_upward_message_num_per_candidate , v12.max_upward_message_num_per_candidate); + assert_eq!(v11.hrmp_max_message_num_per_candidate , v12.hrmp_max_message_num_per_candidate); + assert_eq!(v11.validation_upgrade_cooldown , v12.validation_upgrade_cooldown); + assert_eq!(v11.validation_upgrade_delay , v12.validation_upgrade_delay); + assert_eq!(v11.max_pov_size , v12.max_pov_size); + assert_eq!(v11.max_downward_message_size , v12.max_downward_message_size); + assert_eq!(v11.hrmp_max_parachain_outbound_channels , v12.hrmp_max_parachain_outbound_channels); + assert_eq!(v11.hrmp_sender_deposit , v12.hrmp_sender_deposit); + assert_eq!(v11.hrmp_recipient_deposit , v12.hrmp_recipient_deposit); + assert_eq!(v11.hrmp_channel_max_capacity , v12.hrmp_channel_max_capacity); + assert_eq!(v11.hrmp_channel_max_total_size , v12.hrmp_channel_max_total_size); + assert_eq!(v11.hrmp_max_parachain_inbound_channels , v12.hrmp_max_parachain_inbound_channels); + assert_eq!(v11.hrmp_channel_max_message_size , v12.hrmp_channel_max_message_size); + assert_eq!(v11.code_retention_period , v12.code_retention_period); + assert_eq!(v11.max_validators , v12.max_validators); + assert_eq!(v11.dispute_period , v12.dispute_period); + assert_eq!(v11.no_show_slots , v12.no_show_slots); + assert_eq!(v11.n_delay_tranches , v12.n_delay_tranches); + assert_eq!(v11.zeroth_delay_tranche_width , v12.zeroth_delay_tranche_width); + assert_eq!(v11.needed_approvals , v12.needed_approvals); + assert_eq!(v11.relay_vrf_modulo_samples , v12.relay_vrf_modulo_samples); + assert_eq!(v11.pvf_voting_ttl , v12.pvf_voting_ttl); + assert_eq!(v11.minimum_validation_upgrade_delay , v12.minimum_validation_upgrade_delay); + assert_eq!(v11.async_backing_params.allowed_ancestry_len, v12.async_backing_params.allowed_ancestry_len); + assert_eq!(v11.async_backing_params.max_candidate_depth , v12.async_backing_params.max_candidate_depth); + assert_eq!(v11.executor_params , v12.executor_params); + assert_eq!(v11.minimum_backing_votes , v12.minimum_backing_votes); + assert_eq!(v11.group_rotation_frequency , v12.scheduler_params.group_rotation_frequency); + assert_eq!(v11.paras_availability_period , v12.scheduler_params.paras_availability_period); + assert_eq!(v11.max_validators_per_core , v12.scheduler_params.max_validators_per_core); + assert_eq!(v11.scheduling_lookahead , v12.scheduler_params.lookahead); + assert_eq!(v11.coretime_cores , v12.scheduler_params.num_cores); + assert_eq!(v11.on_demand_retries , v12.scheduler_params.max_availability_timeouts); + assert_eq!(v11.on_demand_queue_max_size , v12.scheduler_params.on_demand_queue_max_size); + assert_eq!(v11.on_demand_target_queue_utilization , v12.scheduler_params.on_demand_target_queue_utilization); + assert_eq!(v11.on_demand_fee_variability , v12.scheduler_params.on_demand_fee_variability); + assert_eq!(v11.on_demand_base_fee , v12.scheduler_params.on_demand_base_fee); + assert_eq!(v11.on_demand_ttl , v12.scheduler_params.ttl); + }; // ; makes this a statement. `rustfmt::skip` cannot be put on an expression. + } + }); + } + + // Test that migration doesn't panic in case there are no pending configurations upgrades in + // pallet's storage. + #[test] + fn test_migrate_to_v12_no_pending() { + let v11 = V11HostConfiguration::::default(); + + new_test_ext(Default::default()).execute_with(|| { + // Implant the v10 version in the state. + v11::ActiveConfig::::set(Some(v11)); + // Ensure there are no pending configs. + v12::PendingConfigs::::set(None); + + // Shouldn't fail. + migrate_to_v12::(); + }); + } +} diff --git a/polkadot/runtime/parachains/src/configuration/tests.rs b/polkadot/runtime/parachains/src/configuration/tests.rs index f1570017dd7b..254511231cac 100644 --- a/polkadot/runtime/parachains/src/configuration/tests.rs +++ b/polkadot/runtime/parachains/src/configuration/tests.rs @@ -226,8 +226,11 @@ fn invariants() { ); ActiveConfig::::put(HostConfiguration { - paras_availability_period: 10, minimum_validation_upgrade_delay: 11, + scheduler_params: SchedulerParams { + paras_availability_period: 10, + ..Default::default() + }, ..Default::default() }); assert_err!( @@ -283,12 +286,6 @@ fn setting_pending_config_members() { max_code_size: 100_000, max_pov_size: 1024, max_head_data_size: 1_000, - coretime_cores: 2, - on_demand_retries: 5, - group_rotation_frequency: 20, - paras_availability_period: 10, - scheduling_lookahead: 3, - max_validators_per_core: None, max_validators: None, dispute_period: 239, dispute_post_conclusion_acceptance_period: 10, @@ -314,13 +311,21 @@ fn setting_pending_config_members() { minimum_validation_upgrade_delay: 20, executor_params: Default::default(), approval_voting_params: ApprovalVotingParams { max_approval_coalesce_count: 1 }, - on_demand_queue_max_size: 10_000u32, - on_demand_base_fee: 10_000_000u128, - on_demand_fee_variability: Perbill::from_percent(3), - on_demand_target_queue_utilization: Perbill::from_percent(25), - on_demand_ttl: 5u32, minimum_backing_votes: 5, node_features: bitvec![u8, Lsb0; 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1], + scheduler_params: SchedulerParams { + group_rotation_frequency: 20, + paras_availability_period: 10, + max_validators_per_core: None, + lookahead: 3, + num_cores: 2, + max_availability_timeouts: 5, + on_demand_queue_max_size: 10_000u32, + on_demand_base_fee: 10_000_000u128, + on_demand_fee_variability: Perbill::from_percent(3), + on_demand_target_queue_utilization: Perbill::from_percent(25), + ttl: 5u32, + }, }; Configuration::set_validation_upgrade_cooldown( @@ -342,13 +347,19 @@ fn setting_pending_config_members() { Configuration::set_max_pov_size(RuntimeOrigin::root(), new_config.max_pov_size).unwrap(); Configuration::set_max_head_data_size(RuntimeOrigin::root(), new_config.max_head_data_size) .unwrap(); - Configuration::set_coretime_cores(RuntimeOrigin::root(), new_config.coretime_cores) - .unwrap(); - Configuration::set_on_demand_retries(RuntimeOrigin::root(), new_config.on_demand_retries) - .unwrap(); + Configuration::set_coretime_cores( + RuntimeOrigin::root(), + new_config.scheduler_params.num_cores, + ) + .unwrap(); + Configuration::set_max_availability_timeouts( + RuntimeOrigin::root(), + new_config.scheduler_params.max_availability_timeouts, + ) + .unwrap(); Configuration::set_group_rotation_frequency( RuntimeOrigin::root(), - new_config.group_rotation_frequency, + new_config.scheduler_params.group_rotation_frequency, ) .unwrap(); // This comes out of order to satisfy the validity criteria for the chain and thread @@ -360,17 +371,17 @@ fn setting_pending_config_members() { .unwrap(); Configuration::set_paras_availability_period( RuntimeOrigin::root(), - new_config.paras_availability_period, + new_config.scheduler_params.paras_availability_period, ) .unwrap(); Configuration::set_scheduling_lookahead( RuntimeOrigin::root(), - new_config.scheduling_lookahead, + new_config.scheduler_params.lookahead, ) .unwrap(); Configuration::set_max_validators_per_core( RuntimeOrigin::root(), - new_config.max_validators_per_core, + new_config.scheduler_params.max_validators_per_core, ) .unwrap(); Configuration::set_max_validators(RuntimeOrigin::root(), new_config.max_validators) diff --git a/polkadot/runtime/parachains/src/coretime/migration.rs b/polkadot/runtime/parachains/src/coretime/migration.rs index 9bc0a20ef5b4..03fecc58570f 100644 --- a/polkadot/runtime/parachains/src/coretime/migration.rs +++ b/polkadot/runtime/parachains/src/coretime/migration.rs @@ -114,7 +114,7 @@ mod v_coretime { let legacy_paras = paras::Parachains::::get(); let config = >::config(); - let total_core_count = config.coretime_cores + legacy_paras.len() as u32; + let total_core_count = config.scheduler_params.num_cores + legacy_paras.len() as u32; let dmp_queue_size = crate::dmp::Pallet::::dmq_contents(T::BrokerId::get().into()).len() as u32; @@ -150,7 +150,7 @@ mod v_coretime { // Migrate to Coretime. // - // NOTE: Also migrates coretime_cores config value in configuration::ActiveConfig. + // NOTE: Also migrates `num_cores` config value in configuration::ActiveConfig. fn migrate_to_coretime< T: Config, SendXcm: xcm::v4::SendXcm, @@ -176,8 +176,8 @@ mod v_coretime { } let config = >::config(); - // coretime_cores was on_demand_cores until now: - for on_demand in 0..config.coretime_cores { + // num_cores was on_demand_cores until now: + for on_demand in 0..config.scheduler_params.num_cores { let core = CoreIndex(legacy_count.saturating_add(on_demand as _)); let r = assigner_coretime::Pallet::::assign_core( core, @@ -189,9 +189,9 @@ mod v_coretime { log::error!("Creating assignment for existing on-demand core, failed: {:?}", err); } } - let total_cores = config.coretime_cores + legacy_count; + let total_cores = config.scheduler_params.num_cores + legacy_count; configuration::ActiveConfig::::mutate(|c| { - c.coretime_cores = total_cores; + c.scheduler_params.num_cores = total_cores; }); if let Err(err) = migrate_send_assignments_to_coretime_chain::() { @@ -200,7 +200,9 @@ mod v_coretime { let single_weight = ::WeightInfo::assign_core(1); single_weight - .saturating_mul(u64::from(legacy_count.saturating_add(config.coretime_cores))) + .saturating_mul(u64::from( + legacy_count.saturating_add(config.scheduler_params.num_cores), + )) // Second read from sending assignments to the coretime chain. .saturating_add(T::DbWeight::get().reads_writes(2, 1)) } @@ -244,7 +246,8 @@ mod v_coretime { Some(mk_coretime_call(crate::coretime::CoretimeCalls::SetLease(p.into(), time_slice))) }); - let core_count: u16 = configuration::Pallet::::config().coretime_cores.saturated_into(); + let core_count: u16 = + configuration::Pallet::::config().scheduler_params.num_cores.saturated_into(); let set_core_count = iter::once(mk_coretime_call( crate::coretime::CoretimeCalls::NotifyCoreCount(core_count), )); diff --git a/polkadot/runtime/parachains/src/coretime/mod.rs b/polkadot/runtime/parachains/src/coretime/mod.rs index 531f5c2e4e47..eb9646d7e869 100644 --- a/polkadot/runtime/parachains/src/coretime/mod.rs +++ b/polkadot/runtime/parachains/src/coretime/mod.rs @@ -214,8 +214,8 @@ impl Pallet { } pub fn initializer_on_new_session(notification: &SessionChangeNotification>) { - let old_core_count = notification.prev_config.coretime_cores; - let new_core_count = notification.new_config.coretime_cores; + let old_core_count = notification.prev_config.scheduler_params.num_cores; + let new_core_count = notification.new_config.scheduler_params.num_cores; if new_core_count != old_core_count { let core_count: u16 = new_core_count.saturated_into(); let message = Xcm(vec![ diff --git a/polkadot/runtime/parachains/src/inclusion/tests.rs b/polkadot/runtime/parachains/src/inclusion/tests.rs index d2b5a67c3e45..3fe7d7f0c7d4 100644 --- a/polkadot/runtime/parachains/src/inclusion/tests.rs +++ b/polkadot/runtime/parachains/src/inclusion/tests.rs @@ -47,10 +47,10 @@ use test_helpers::{dummy_collator, dummy_collator_signature, dummy_validation_co fn default_config() -> HostConfiguration { let mut config = HostConfiguration::default(); - config.coretime_cores = 1; + config.scheduler_params.num_cores = 1; config.max_code_size = 0b100000; config.max_head_data_size = 0b100000; - config.group_rotation_frequency = u32::MAX; + config.scheduler_params.group_rotation_frequency = u32::MAX; config } @@ -224,7 +224,7 @@ pub(crate) fn run_to_block( } pub(crate) fn expected_bits() -> usize { - Paras::parachains().len() + Configuration::config().coretime_cores as usize + Paras::parachains().len() + Configuration::config().scheduler_params.num_cores as usize } fn default_bitfield() -> AvailabilityBitfield { @@ -386,7 +386,7 @@ fn collect_pending_cleans_up_pending() { (thread_a, ParaKind::Parathread), ]; let mut config = genesis_config(paras); - config.configuration.config.group_rotation_frequency = 3; + config.configuration.config.scheduler_params.group_rotation_frequency = 3; new_test_ext(config).execute_with(|| { let default_candidate = TestCandidateBuilder::default().build(); >::insert( @@ -2062,7 +2062,7 @@ fn check_allowed_relay_parents() { } let validator_public = validator_pubkeys(&validators); let mut config = genesis_config(paras); - config.configuration.config.group_rotation_frequency = 1; + config.configuration.config.scheduler_params.group_rotation_frequency = 1; new_test_ext(config).execute_with(|| { shared::Pallet::::set_active_validators_ascending(validator_public.clone()); diff --git a/polkadot/runtime/parachains/src/mock.rs b/polkadot/runtime/parachains/src/mock.rs index 1925ca19501a..e18be03bdeb2 100644 --- a/polkadot/runtime/parachains/src/mock.rs +++ b/polkadot/runtime/parachains/src/mock.rs @@ -23,7 +23,7 @@ use crate::{ initializer, origin, paras, paras::ParaKind, paras_inherent, scheduler, - scheduler::common::{AssignmentProvider, AssignmentProviderConfig}, + scheduler::common::AssignmentProvider, session_info, shared, ParaId, }; use frame_support::pallet_prelude::*; @@ -463,10 +463,6 @@ pub mod mock_assigner { pub(super) type MockAssignmentQueue = StorageValue<_, VecDeque, ValueQuery>; - #[pallet::storage] - pub(super) type MockProviderConfig = - StorageValue<_, AssignmentProviderConfig, OptionQuery>; - #[pallet::storage] pub(super) type MockCoreCount = StorageValue<_, u32, OptionQuery>; } @@ -478,12 +474,6 @@ pub mod mock_assigner { MockAssignmentQueue::::mutate(|queue| queue.push_back(assignment)); } - // This configuration needs to be customized to service `get_provider_config` in - // scheduler tests. - pub fn set_assignment_provider_config(config: AssignmentProviderConfig) { - MockProviderConfig::::set(Some(config)); - } - // Allows for customized core count in scheduler tests, rather than a core count // derived from on-demand config + parachain count. pub fn set_core_count(count: u32) { @@ -512,17 +502,6 @@ pub mod mock_assigner { // in the mock assigner. fn push_back_assignment(_assignment: Assignment) {} - // Gets the provider config we set earlier using `set_assignment_provider_config`, falling - // back to the on demand parachain configuration if none was set. - fn get_provider_config(_core_idx: CoreIndex) -> AssignmentProviderConfig { - match MockProviderConfig::::get() { - Some(config) => config, - None => AssignmentProviderConfig { - max_availability_timeouts: 1, - ttl: BlockNumber::from(5u32), - }, - } - } #[cfg(any(feature = "runtime-benchmarks", test))] fn get_mock_assignment(_: CoreIndex, para_id: ParaId) -> Assignment { Assignment::Bulk(para_id) diff --git a/polkadot/runtime/parachains/src/paras/tests.rs b/polkadot/runtime/parachains/src/paras/tests.rs index 24ea919ec875..262ec9d3fdba 100644 --- a/polkadot/runtime/parachains/src/paras/tests.rs +++ b/polkadot/runtime/parachains/src/paras/tests.rs @@ -17,7 +17,7 @@ use super::*; use frame_support::{assert_err, assert_ok, assert_storage_noop}; use keyring::Sr25519Keyring; -use primitives::{BlockNumber, PARACHAIN_KEY_TYPE_ID}; +use primitives::{vstaging::SchedulerParams, BlockNumber, PARACHAIN_KEY_TYPE_ID}; use sc_keystore::LocalKeystore; use sp_keystore::{Keystore, KeystorePtr}; use std::sync::Arc; @@ -909,7 +909,10 @@ fn full_parachain_cleanup_storage() { minimum_validation_upgrade_delay: 2, // Those are not relevant to this test. However, HostConfiguration is still a // subject for the consistency check. - paras_availability_period: 1, + scheduler_params: SchedulerParams { + paras_availability_period: 1, + ..Default::default() + }, ..Default::default() }, }, diff --git a/polkadot/runtime/parachains/src/paras_inherent/tests.rs b/polkadot/runtime/parachains/src/paras_inherent/tests.rs index defb2f4404f5..b7285ec884ad 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/tests.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/tests.rs @@ -27,13 +27,14 @@ mod enter { builder::{Bench, BenchBuilder}, mock::{mock_assigner, new_test_ext, BlockLength, BlockWeights, MockGenesisConfig, Test}, scheduler::{ - common::{Assignment, AssignmentProvider, AssignmentProviderConfig}, + common::{Assignment, AssignmentProvider}, ParasEntry, }, }; use assert_matches::assert_matches; use frame_support::assert_ok; use frame_system::limits; + use primitives::vstaging::SchedulerParams; use sp_runtime::Perbill; use sp_std::collections::btree_map::BTreeMap; @@ -87,7 +88,7 @@ mod enter { // `create_inherent` and will not cause `enter` to early. fn include_backed_candidates() { let config = MockGenesisConfig::default(); - assert!(config.configuration.config.scheduling_lookahead > 0); + assert!(config.configuration.config.scheduler_params.lookahead > 0); new_test_ext(config).execute_with(|| { let dispute_statements = BTreeMap::new(); @@ -625,7 +626,7 @@ mod enter { #[test] fn limit_candidates_over_weight_1() { let config = MockGenesisConfig::default(); - assert!(config.configuration.config.scheduling_lookahead > 0); + assert!(config.configuration.config.scheduler_params.lookahead > 0); new_test_ext(config).execute_with(|| { // Create the inherent data for this block @@ -706,8 +707,8 @@ mod enter { let cores = (0..used_cores) .into_iter() .map(|i| { - let AssignmentProviderConfig { ttl, .. } = - scheduler::Pallet::::assignment_provider_config(CoreIndex(i)); + let SchedulerParams { ttl, .. } = + >::config().scheduler_params; // Load an assignment into provider so that one is present to pop let assignment = ::AssignmentProvider::get_mock_assignment( diff --git a/polkadot/runtime/parachains/src/scheduler.rs b/polkadot/runtime/parachains/src/scheduler.rs index 3dbb050fb4e5..61ab36dbe96c 100644 --- a/polkadot/runtime/parachains/src/scheduler.rs +++ b/polkadot/runtime/parachains/src/scheduler.rs @@ -51,7 +51,7 @@ use sp_std::{ pub mod common; -use common::{Assignment, AssignmentProvider, AssignmentProviderConfig}; +use common::{Assignment, AssignmentProvider}; pub use pallet::*; @@ -222,7 +222,7 @@ impl Pallet { let n_cores = core::cmp::max( T::AssignmentProvider::session_core_count(), - match config.max_validators_per_core { + match config.scheduler_params.max_validators_per_core { Some(x) if x != 0 => validators.len() as u32 / x, _ => 0, }, @@ -350,6 +350,7 @@ impl Pallet { fn drop_expired_claims_from_claimqueue() { let now = >::block_number(); let availability_cores = AvailabilityCores::::get(); + let ttl = >::config().scheduler_params.ttl; ClaimQueue::::mutate(|cq| { for (idx, _) in (0u32..).zip(availability_cores) { @@ -382,8 +383,6 @@ impl Pallet { if let Some(assignment) = T::AssignmentProvider::pop_assignment_for_core(core_idx) { - let AssignmentProviderConfig { ttl, .. } = - T::AssignmentProvider::get_provider_config(core_idx); core_claimqueue.push_back(ParasEntry::new(assignment, now + ttl)); } } @@ -428,7 +427,7 @@ impl Pallet { } let rotations_since_session_start: BlockNumberFor = - (at - session_start_block) / config.group_rotation_frequency; + (at - session_start_block) / config.scheduler_params.group_rotation_frequency; let rotations_since_session_start = as TryInto>::try_into(rotations_since_session_start) @@ -460,9 +459,9 @@ impl Pallet { // Note: blocks backed in this rotation will never time out here as backed_in + // config.paras_availability_period will always be > now for these blocks, as // otherwise above condition would not be true. - pending_since + config.paras_availability_period + pending_since + config.scheduler_params.paras_availability_period } else { - next_rotation + config.paras_availability_period + next_rotation + config.scheduler_params.paras_availability_period }; AvailabilityTimeoutStatus { timed_out: time_out_at <= now, live_until: time_out_at } @@ -478,7 +477,8 @@ impl Pallet { let now = >::block_number() + One::one(); let rotation_info = Self::group_rotation_info(now); - let current_window = rotation_info.last_rotation_at() + config.paras_availability_period; + let current_window = + rotation_info.last_rotation_at() + config.scheduler_params.paras_availability_period; now < current_window } @@ -488,7 +488,7 @@ impl Pallet { ) -> GroupRotationInfo> { let session_start_block = Self::session_start_block(); let group_rotation_frequency = - >::config().group_rotation_frequency; + >::config().scheduler_params.group_rotation_frequency; GroupRotationInfo { session_start_block, now, group_rotation_frequency } } @@ -508,6 +508,8 @@ impl Pallet { /// Return the next thing that will be scheduled on this core assuming it is currently /// occupied and the candidate occupying it times out. pub(crate) fn next_up_on_time_out(core: CoreIndex) -> Option { + let max_availability_timeouts = + >::config().scheduler_params.max_availability_timeouts; Self::next_up_on_available(core).or_else(|| { // Or, if none, the claim currently occupying the core, // as it would be put back on the queue after timing out if number of retries is not at @@ -515,16 +517,12 @@ impl Pallet { let cores = AvailabilityCores::::get(); cores.get(core.0 as usize).and_then(|c| match c { CoreOccupied::Free => None, - CoreOccupied::Paras(pe) => { - let AssignmentProviderConfig { max_availability_timeouts, .. } = - T::AssignmentProvider::get_provider_config(core); - + CoreOccupied::Paras(pe) => if pe.availability_timeouts < max_availability_timeouts { Some(Self::paras_entry_to_scheduled_core(pe)) } else { None - } - }, + }, }) }) } @@ -566,7 +564,7 @@ impl Pallet { // ClaimQueue related functions // fn claimqueue_lookahead() -> u32 { - >::config().scheduling_lookahead + >::config().scheduler_params.lookahead } /// Frees cores and fills the free claimqueue spots by popping from the `AssignmentProvider`. @@ -585,15 +583,15 @@ impl Pallet { let n_lookahead = Self::claimqueue_lookahead().max(1); let n_session_cores = T::AssignmentProvider::session_core_count(); let cq = ClaimQueue::::get(); - let ttl = >::config().on_demand_ttl; + let config = >::config(); + let max_availability_timeouts = config.scheduler_params.max_availability_timeouts; + let ttl = config.scheduler_params.ttl; for core_idx in 0..n_session_cores { let core_idx = CoreIndex::from(core_idx); // add previously timedout paras back into the queue if let Some(mut entry) = timedout_paras.remove(&core_idx) { - let AssignmentProviderConfig { max_availability_timeouts, .. } = - T::AssignmentProvider::get_provider_config(core_idx); if entry.availability_timeouts < max_availability_timeouts { // Increment the timeout counter. entry.availability_timeouts += 1; @@ -668,13 +666,6 @@ impl Pallet { .filter_map(|(core_idx, v)| v.front().map(|e| (core_idx, e.assignment.para_id()))) } - #[cfg(any(feature = "runtime-benchmarks", test))] - pub(crate) fn assignment_provider_config( - core_idx: CoreIndex, - ) -> AssignmentProviderConfig> { - T::AssignmentProvider::get_provider_config(core_idx) - } - #[cfg(any(feature = "try-runtime", test))] fn claimqueue_len() -> usize { ClaimQueue::::get().iter().map(|la_vec| la_vec.1.len()).sum() diff --git a/polkadot/runtime/parachains/src/scheduler/common.rs b/polkadot/runtime/parachains/src/scheduler/common.rs index 2eb73385803c..66a4e6d30be0 100644 --- a/polkadot/runtime/parachains/src/scheduler/common.rs +++ b/polkadot/runtime/parachains/src/scheduler/common.rs @@ -48,22 +48,10 @@ impl Assignment { } } -#[derive(Encode, Decode, TypeInfo)] -/// A set of variables required by the scheduler in order to operate. -pub struct AssignmentProviderConfig { - /// How many times a collation can time out on availability. - /// Zero timeouts still means that a collation can be provided as per the slot auction - /// assignment provider. - pub max_availability_timeouts: u32, - - /// How long the collator has to provide a collation to the backing group before being dropped. - pub ttl: BlockNumber, -} - pub trait AssignmentProvider { /// Pops an [`Assignment`] from the provider for a specified [`CoreIndex`]. /// - /// This is where assignments come into existance. + /// This is where assignments come into existence. fn pop_assignment_for_core(core_idx: CoreIndex) -> Option; /// A previously popped `Assignment` has been fully processed. @@ -77,14 +65,11 @@ pub trait AssignmentProvider { /// Push back a previously popped assignment. /// /// If the assignment could not be processed within the current session, it can be pushed back - /// to the assignment provider in order to be poppped again later. + /// to the assignment provider in order to be popped again later. /// /// This is the second way the life of an assignment can come to an end. fn push_back_assignment(assignment: Assignment); - /// Returns a set of variables needed by the scheduler - fn get_provider_config(core_idx: CoreIndex) -> AssignmentProviderConfig; - /// Push some assignment for mocking/benchmarks purposes. /// /// Useful for benchmarks and testing. The returned assignment is "valid" and can if need be diff --git a/polkadot/runtime/parachains/src/scheduler/tests.rs b/polkadot/runtime/parachains/src/scheduler/tests.rs index 9af23ce64bd6..28b3a6b4f5d6 100644 --- a/polkadot/runtime/parachains/src/scheduler/tests.rs +++ b/polkadot/runtime/parachains/src/scheduler/tests.rs @@ -18,7 +18,9 @@ use super::*; use frame_support::assert_ok; use keyring::Sr25519Keyring; -use primitives::{BlockNumber, SessionIndex, ValidationCode, ValidatorId}; +use primitives::{ + vstaging::SchedulerParams, BlockNumber, SessionIndex, ValidationCode, ValidatorId, +}; use sp_std::collections::{btree_map::BTreeMap, btree_set::BTreeSet}; use crate::{ @@ -103,15 +105,19 @@ fn run_to_end_of_block( fn default_config() -> HostConfiguration { HostConfiguration { - coretime_cores: 3, - group_rotation_frequency: 10, - paras_availability_period: 3, - scheduling_lookahead: 2, // This field does not affect anything that scheduler does. However, `HostConfiguration` // is still a subject to consistency test. It requires that // `minimum_validation_upgrade_delay` is greater than `chain_availability_period` and // `thread_availability_period`. minimum_validation_upgrade_delay: 6, + scheduler_params: SchedulerParams { + group_rotation_frequency: 10, + paras_availability_period: 3, + lookahead: 2, + num_cores: 3, + max_availability_timeouts: 1, + ..Default::default() + }, ..Default::default() } } @@ -155,7 +161,7 @@ fn scheduled_entries() -> impl Iterator(vec![para_a])); } else { assert!(!claimqueue_contains_para_ids::(vec![para_a])); @@ -822,7 +825,7 @@ fn on_demand_claims_are_pruned_after_timing_out() { assert!(!availability_cores_contains_para_ids::(vec![para_a])); // #25 - now += max_retries + 2; + now += max_timeouts + 2; // Add assignment back to the mix. MockAssigner::add_test_assignment(assignment_a.clone()); @@ -833,7 +836,7 @@ fn on_demand_claims_are_pruned_after_timing_out() { // #26 now += 1; // Run to block #n but this time have group 1 conclude the availabilty. - for n in now..=(now + max_retries + 1) { + for n in now..=(now + max_timeouts + 1) { // #n run_to_block(n, |_| None); // Time out core 0 if group 0 is assigned to it, if group 1 is assigned, conclude. @@ -874,10 +877,8 @@ fn on_demand_claims_are_pruned_after_timing_out() { fn availability_predicate_works() { let genesis_config = genesis_config(&default_config()); - let HostConfiguration { group_rotation_frequency, paras_availability_period, .. } = - default_config(); - - assert!(paras_availability_period < group_rotation_frequency); + let SchedulerParams { group_rotation_frequency, paras_availability_period, .. } = + default_config().scheduler_params; new_test_ext(genesis_config).execute_with(|| { run_to_block(1 + paras_availability_period, |_| None); @@ -1044,7 +1045,7 @@ fn next_up_on_time_out_reuses_claim_if_nothing_queued() { #[test] fn session_change_requires_reschedule_dropping_removed_paras() { let mut config = default_config(); - config.scheduling_lookahead = 1; + config.scheduler_params.lookahead = 1; let genesis_config = genesis_config(&config); let para_a = ParaId::from(1_u32); @@ -1056,7 +1057,7 @@ fn session_change_requires_reschedule_dropping_removed_paras() { new_test_ext(genesis_config).execute_with(|| { // Setting explicit core count MockAssigner::set_core_count(5); - let assignment_provider_ttl = MockAssigner::get_provider_config(CoreIndex::from(0)).ttl; + let coretime_ttl = >::config().scheduler_params.ttl; schedule_blank_para(para_a); schedule_blank_para(para_b); @@ -1120,7 +1121,7 @@ fn session_change_requires_reschedule_dropping_removed_paras() { vec![ParasEntry::new( Assignment::Bulk(para_a), // At end of block 2 - assignment_provider_ttl + 2 + coretime_ttl + 2 )] .into_iter() .collect() @@ -1169,7 +1170,7 @@ fn session_change_requires_reschedule_dropping_removed_paras() { vec![ParasEntry::new( Assignment::Bulk(para_a), // At block 3 - assignment_provider_ttl + 3 + coretime_ttl + 3 )] .into_iter() .collect() @@ -1179,7 +1180,7 @@ fn session_change_requires_reschedule_dropping_removed_paras() { vec![ParasEntry::new( Assignment::Bulk(para_b), // At block 3 - assignment_provider_ttl + 3 + coretime_ttl + 3 )] .into_iter() .collect() diff --git a/polkadot/runtime/parachains/src/session_info/tests.rs b/polkadot/runtime/parachains/src/session_info/tests.rs index 92a50575deda..a5bfeae07455 100644 --- a/polkadot/runtime/parachains/src/session_info/tests.rs +++ b/polkadot/runtime/parachains/src/session_info/tests.rs @@ -25,7 +25,7 @@ use crate::{ util::take_active_subset, }; use keyring::Sr25519Keyring; -use primitives::{BlockNumber, ValidatorId, ValidatorIndex}; +use primitives::{vstaging::SchedulerParams, BlockNumber, ValidatorId, ValidatorIndex}; fn run_to_block( to: BlockNumber, @@ -62,9 +62,9 @@ fn run_to_block( fn default_config() -> HostConfiguration { HostConfiguration { - coretime_cores: 1, dispute_period: 2, needed_approvals: 3, + scheduler_params: SchedulerParams { num_cores: 1, ..Default::default() }, ..Default::default() } } diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs index bbc79a13d37f..f3c0c3d6bd8c 100644 --- a/polkadot/runtime/rococo/src/lib.rs +++ b/polkadot/runtime/rococo/src/lib.rs @@ -1658,6 +1658,7 @@ pub mod migrations { parachains_configuration::migration::v11::MigrateToV11, // This needs to come after the `parachains_configuration` above as we are reading the configuration. coretime::migration::MigrateToCoretime, + parachains_configuration::migration::v12::MigrateToV12, // permanent pallet_xcm::migration::MigrateToLatestXcmVersion, @@ -1768,7 +1769,7 @@ sp_api::impl_runtime_apis! { Executive::execute_block(block); } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/polkadot/runtime/rococo/src/weights/runtime_parachains_configuration.rs b/polkadot/runtime/rococo/src/weights/runtime_parachains_configuration.rs index 34541b83597e..ca0575cb1b64 100644 --- a/polkadot/runtime/rococo/src/weights/runtime_parachains_configuration.rs +++ b/polkadot/runtime/rococo/src/weights/runtime_parachains_configuration.rs @@ -16,10 +16,10 @@ //! Autogenerated weights for `runtime_parachains::configuration` //! -//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2023-11-10, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 32.0.0 +//! DATE: 2024-02-08, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` //! WORST CASE MAP SIZE: `1000000` -//! HOSTNAME: `runner-yprdrvc7-project-674-concurrent-0`, CPU: `Intel(R) Xeon(R) CPU @ 2.60GHz` +//! HOSTNAME: `runner-bn-ce5rx-project-674-concurrent-0`, CPU: `Intel(R) Xeon(R) CPU @ 2.60GHz` //! WASM-EXECUTION: `Compiled`, CHAIN: `Some("rococo-dev")`, DB CACHE: 1024 // Executed Command: @@ -58,8 +58,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `151` // Estimated: `1636` - // Minimum execution time: 7_793_000 picoseconds. - Weight::from_parts(8_192_000, 0) + // Minimum execution time: 7_789_000 picoseconds. + Weight::from_parts(8_269_000, 0) .saturating_add(Weight::from_parts(0, 1636)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -74,8 +74,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `151` // Estimated: `1636` - // Minimum execution time: 7_819_000 picoseconds. - Weight::from_parts(8_004_000, 0) + // Minimum execution time: 7_851_000 picoseconds. + Weight::from_parts(8_152_000, 0) .saturating_add(Weight::from_parts(0, 1636)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -90,8 +90,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `151` // Estimated: `1636` - // Minimum execution time: 7_760_000 picoseconds. - Weight::from_parts(8_174_000, 0) + // Minimum execution time: 7_960_000 picoseconds. + Weight::from_parts(8_276_000, 0) .saturating_add(Weight::from_parts(0, 1636)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -116,8 +116,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `151` // Estimated: `1636` - // Minimum execution time: 7_814_000 picoseconds. - Weight::from_parts(8_098_000, 0) + // Minimum execution time: 7_912_000 picoseconds. + Weight::from_parts(8_164_000, 0) .saturating_add(Weight::from_parts(0, 1636)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -132,8 +132,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `151` // Estimated: `1636` - // Minimum execution time: 10_028_000 picoseconds. - Weight::from_parts(10_386_000, 0) + // Minimum execution time: 9_782_000 picoseconds. + Weight::from_parts(10_373_000, 0) .saturating_add(Weight::from_parts(0, 1636)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -148,8 +148,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `151` // Estimated: `1636` - // Minimum execution time: 7_867_000 picoseconds. - Weight::from_parts(8_191_000, 0) + // Minimum execution time: 7_870_000 picoseconds. + Weight::from_parts(8_274_000, 0) .saturating_add(Weight::from_parts(0, 1636)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -164,8 +164,24 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `151` // Estimated: `1636` - // Minimum execution time: 10_158_000 picoseconds. - Weight::from_parts(10_430_000, 0) + // Minimum execution time: 9_960_000 picoseconds. + Weight::from_parts(10_514_000, 0) + .saturating_add(Weight::from_parts(0, 1636)) + .saturating_add(T::DbWeight::get().reads(3)) + .saturating_add(T::DbWeight::get().writes(1)) + } + /// Storage: `Configuration::PendingConfigs` (r:1 w:1) + /// Proof: `Configuration::PendingConfigs` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `Configuration::BypassConsistencyCheck` (r:1 w:0) + /// Proof: `Configuration::BypassConsistencyCheck` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `ParasShared::CurrentSessionIndex` (r:1 w:0) + /// Proof: `ParasShared::CurrentSessionIndex` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + fn set_config_with_scheduler_params() -> Weight { + // Proof Size summary in bytes: + // Measured: `151` + // Estimated: `1636` + // Minimum execution time: 7_913_000 picoseconds. + Weight::from_parts(8_338_000, 0) .saturating_add(Weight::from_parts(0, 1636)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) diff --git a/polkadot/runtime/test-runtime/src/lib.rs b/polkadot/runtime/test-runtime/src/lib.rs index b74def5de8aa..6c899c527015 100644 --- a/polkadot/runtime/test-runtime/src/lib.rs +++ b/polkadot/runtime/test-runtime/src/lib.rs @@ -767,7 +767,7 @@ sp_api::impl_runtime_apis! { Executive::execute_block(block); } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs index 574710b3110d..a7772303a95d 100644 --- a/polkadot/runtime/westend/src/lib.rs +++ b/polkadot/runtime/westend/src/lib.rs @@ -1699,6 +1699,7 @@ pub mod migrations { // Migrate Identity pallet for Usernames pallet_identity::migration::versioned::V0ToV1, parachains_configuration::migration::v11::MigrateToV11, + parachains_configuration::migration::v12::MigrateToV12, // permanent pallet_xcm::migration::MigrateToLatestXcmVersion, // Migrate from legacy lease to coretime. Needs to run after configuration v11 @@ -1793,7 +1794,7 @@ sp_api::impl_runtime_apis! { Executive::execute_block(block); } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/polkadot/runtime/westend/src/weights/runtime_parachains_configuration.rs b/polkadot/runtime/westend/src/weights/runtime_parachains_configuration.rs index 3a4813b667c6..8fa3207c6446 100644 --- a/polkadot/runtime/westend/src/weights/runtime_parachains_configuration.rs +++ b/polkadot/runtime/westend/src/weights/runtime_parachains_configuration.rs @@ -16,10 +16,10 @@ //! Autogenerated weights for `runtime_parachains::configuration` //! -//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2023-11-10, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 32.0.0 +//! DATE: 2024-02-08, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` //! WORST CASE MAP SIZE: `1000000` -//! HOSTNAME: `runner-yprdrvc7-project-674-concurrent-0`, CPU: `Intel(R) Xeon(R) CPU @ 2.60GHz` +//! HOSTNAME: `runner-bn-ce5rx-project-674-concurrent-0`, CPU: `Intel(R) Xeon(R) CPU @ 2.60GHz` //! WASM-EXECUTION: `Compiled`, CHAIN: `Some("westend-dev")`, DB CACHE: 1024 // Executed Command: @@ -58,8 +58,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `151` // Estimated: `1636` - // Minimum execution time: 8_065_000 picoseconds. - Weight::from_parts(8_389_000, 0) + // Minimum execution time: 7_775_000 picoseconds. + Weight::from_parts(8_036_000, 0) .saturating_add(Weight::from_parts(0, 1636)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -74,8 +74,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `151` // Estimated: `1636` - // Minimum execution time: 8_038_000 picoseconds. - Weight::from_parts(8_463_000, 0) + // Minimum execution time: 7_708_000 picoseconds. + Weight::from_parts(7_971_000, 0) .saturating_add(Weight::from_parts(0, 1636)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -90,8 +90,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `151` // Estimated: `1636` - // Minimum execution time: 7_843_000 picoseconds. - Weight::from_parts(8_216_000, 0) + // Minimum execution time: 7_746_000 picoseconds. + Weight::from_parts(8_028_000, 0) .saturating_add(Weight::from_parts(0, 1636)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -116,8 +116,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `151` // Estimated: `1636` - // Minimum execution time: 7_969_000 picoseconds. - Weight::from_parts(8_362_000, 0) + // Minimum execution time: 7_729_000 picoseconds. + Weight::from_parts(7_954_000, 0) .saturating_add(Weight::from_parts(0, 1636)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -132,8 +132,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `151` // Estimated: `1636` - // Minimum execution time: 10_084_000 picoseconds. - Weight::from_parts(10_451_000, 0) + // Minimum execution time: 9_871_000 picoseconds. + Weight::from_parts(10_075_000, 0) .saturating_add(Weight::from_parts(0, 1636)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -148,8 +148,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `151` // Estimated: `1636` - // Minimum execution time: 7_948_000 picoseconds. - Weight::from_parts(8_268_000, 0) + // Minimum execution time: 7_869_000 picoseconds. + Weight::from_parts(8_000_000, 0) .saturating_add(Weight::from_parts(0, 1636)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -164,8 +164,24 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `151` // Estimated: `1636` - // Minimum execution time: 10_257_000 picoseconds. - Weight::from_parts(10_584_000, 0) + // Minimum execution time: 9_797_000 picoseconds. + Weight::from_parts(10_373_000, 0) + .saturating_add(Weight::from_parts(0, 1636)) + .saturating_add(T::DbWeight::get().reads(3)) + .saturating_add(T::DbWeight::get().writes(1)) + } + /// Storage: `Configuration::PendingConfigs` (r:1 w:1) + /// Proof: `Configuration::PendingConfigs` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `Configuration::BypassConsistencyCheck` (r:1 w:0) + /// Proof: `Configuration::BypassConsistencyCheck` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `ParasShared::CurrentSessionIndex` (r:1 w:0) + /// Proof: `ParasShared::CurrentSessionIndex` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + fn set_config_with_scheduler_params() -> Weight { + // Proof Size summary in bytes: + // Measured: `151` + // Estimated: `1636` + // Minimum execution time: 7_718_000 picoseconds. + Weight::from_parts(7_984_000, 0) .saturating_add(Weight::from_parts(0, 1636)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) diff --git a/polkadot/zombienet_tests/functional/0002-parachains-disputes.toml b/polkadot/zombienet_tests/functional/0002-parachains-disputes.toml index f6bdfeb4877e..2561661de1f8 100644 --- a/polkadot/zombienet_tests/functional/0002-parachains-disputes.toml +++ b/polkadot/zombienet_tests/functional/0002-parachains-disputes.toml @@ -2,9 +2,11 @@ timeout = 1000 [relaychain.genesis.runtimeGenesis.patch.configuration.config] - max_validators_per_core = 5 needed_approvals = 8 +[relaychain.genesis.runtimeGenesis.patch.configuration.config.scheduler_params] + max_validators_per_core = 5 + [relaychain.genesis.runtimeGenesis.patch.configuration.config.approval_voting_params] max_approval_coalesce_count = 5 diff --git a/polkadot/zombienet_tests/functional/0004-parachains-garbage-candidate.toml b/polkadot/zombienet_tests/functional/0004-parachains-garbage-candidate.toml index 5d6f299d4613..a2a2621f8426 100644 --- a/polkadot/zombienet_tests/functional/0004-parachains-garbage-candidate.toml +++ b/polkadot/zombienet_tests/functional/0004-parachains-garbage-candidate.toml @@ -2,8 +2,10 @@ timeout = 1000 bootnode = true -[relaychain.genesis.runtimeGenesis.patch.configuration.config] +[relaychain.genesis.runtimeGenesis.patch.configuration.config.scheduler_params] max_validators_per_core = 1 + +[relaychain.genesis.runtimeGenesis.patch.configuration.config] needed_approvals = 2 [relaychain] diff --git a/polkadot/zombienet_tests/functional/0005-parachains-disputes-past-session.toml b/polkadot/zombienet_tests/functional/0005-parachains-disputes-past-session.toml index e2fbec079b1a..a3bbc82e74ba 100644 --- a/polkadot/zombienet_tests/functional/0005-parachains-disputes-past-session.toml +++ b/polkadot/zombienet_tests/functional/0005-parachains-disputes-past-session.toml @@ -3,8 +3,10 @@ timeout = 1000 bootnode = true [relaychain.genesis.runtimeGenesis.patch.configuration.config] - max_validators_per_core = 1 needed_approvals = 2 + +[relaychain.genesis.runtimeGenesis.patch.configuration.config.scheduler_params] + max_validators_per_core = 1 group_rotation_frequency = 2 [relaychain] diff --git a/polkadot/zombienet_tests/functional/0006-parachains-max-tranche0.toml b/polkadot/zombienet_tests/functional/0006-parachains-max-tranche0.toml index bef54cb8ca41..858f87b9cfe5 100644 --- a/polkadot/zombienet_tests/functional/0006-parachains-max-tranche0.toml +++ b/polkadot/zombienet_tests/functional/0006-parachains-max-tranche0.toml @@ -3,10 +3,12 @@ timeout = 1000 bootnode = true [relaychain.genesis.runtimeGenesis.patch.configuration.config] - max_validators_per_core = 1 needed_approvals = 7 relay_vrf_modulo_samples = 5 +[relaychain.genesis.runtimeGenesis.patch.configuration.config.scheduler_params] + max_validators_per_core = 1 + [relaychain] default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" chain = "rococo-local" diff --git a/polkadot/zombienet_tests/functional/0007-dispute-freshly-finalized.toml b/polkadot/zombienet_tests/functional/0007-dispute-freshly-finalized.toml index 69eb0804d8cb..573ccf961385 100644 --- a/polkadot/zombienet_tests/functional/0007-dispute-freshly-finalized.toml +++ b/polkadot/zombienet_tests/functional/0007-dispute-freshly-finalized.toml @@ -2,9 +2,11 @@ timeout = 1000 [relaychain.genesis.runtimeGenesis.patch.configuration.config] - max_validators_per_core = 1 needed_approvals = 1 +[relaychain.genesis.runtimeGenesis.patch.configuration.config.scheduler_params] + max_validators_per_core = 1 + [relaychain] default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" chain = "rococo-local" diff --git a/polkadot/zombienet_tests/functional/0008-dispute-old-finalized.toml b/polkadot/zombienet_tests/functional/0008-dispute-old-finalized.toml index 1ea385c3a42e..ea1c93a1403f 100644 --- a/polkadot/zombienet_tests/functional/0008-dispute-old-finalized.toml +++ b/polkadot/zombienet_tests/functional/0008-dispute-old-finalized.toml @@ -2,9 +2,11 @@ timeout = 1000 [relaychain.genesis.runtimeGenesis.patch.configuration.config] - max_validators_per_core = 1 needed_approvals = 1 +[relaychain.genesis.runtimeGenesis.patch.configuration.config.scheduler_params] + max_validators_per_core = 1 + [relaychain] default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" chain = "rococo-local" diff --git a/polkadot/zombienet_tests/functional/0010-validator-disabling.toml b/polkadot/zombienet_tests/functional/0010-validator-disabling.toml index 6701d60d74d1..c9d79c5f8f23 100644 --- a/polkadot/zombienet_tests/functional/0010-validator-disabling.toml +++ b/polkadot/zombienet_tests/functional/0010-validator-disabling.toml @@ -3,8 +3,10 @@ timeout = 1000 bootnode = true [relaychain.genesis.runtimeGenesis.patch.configuration.config] - max_validators_per_core = 1 needed_approvals = 2 + +[relaychain.genesis.runtimeGenesis.patch.configuration.config.scheduler_params] + max_validators_per_core = 1 group_rotation_frequency = 10 [relaychain] diff --git a/polkadot/zombienet_tests/functional/0011-async-backing-6-seconds-rate.toml b/polkadot/zombienet_tests/functional/0011-async-backing-6-seconds-rate.toml index 5a6832b149be..b776622fdce3 100644 --- a/polkadot/zombienet_tests/functional/0011-async-backing-6-seconds-rate.toml +++ b/polkadot/zombienet_tests/functional/0011-async-backing-6-seconds-rate.toml @@ -8,13 +8,16 @@ chain = "rococo-local" [relaychain.genesis.runtimeGenesis.patch.configuration.config] needed_approvals = 4 relay_vrf_modulo_samples = 6 - scheduling_lookahead = 2 - group_rotation_frequency = 4 [relaychain.genesis.runtimeGenesis.patch.configuration.config.async_backing_params] max_candidate_depth = 3 allowed_ancestry_len = 2 +[relaychain.genesis.runtimeGenesis.patch.configuration.config.scheduler_params] + lookahead = 2 + group_rotation_frequency = 4 + + [relaychain.default_resources] limits = { memory = "4G", cpu = "2" } requests = { memory = "2G", cpu = "1" } diff --git a/polkadot/zombienet_tests/functional/0012-elastic-scaling-mvp.toml b/polkadot/zombienet_tests/functional/0012-elastic-scaling-mvp.toml index 0dfd814e10a5..9b3576eaa3c2 100644 --- a/polkadot/zombienet_tests/functional/0012-elastic-scaling-mvp.toml +++ b/polkadot/zombienet_tests/functional/0012-elastic-scaling-mvp.toml @@ -3,9 +3,11 @@ timeout = 1000 bootnode = true [relaychain.genesis.runtimeGenesis.patch.configuration.config] - max_validators_per_core = 2 needed_approvals = 4 - coretime_cores = 2 + +[relaychain.genesis.runtimeGenesis.patch.configuration.config.scheduler_params] + max_validators_per_core = 2 + num_cores = 2 [relaychain] default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" diff --git a/polkadot/zombienet_tests/functional/0012-elastic-scaling-mvp.zndsl b/polkadot/zombienet_tests/functional/0012-elastic-scaling-mvp.zndsl index a7193c9282b9..edc87c802a09 100644 --- a/polkadot/zombienet_tests/functional/0012-elastic-scaling-mvp.zndsl +++ b/polkadot/zombienet_tests/functional/0012-elastic-scaling-mvp.zndsl @@ -1,4 +1,4 @@ -Description: Test that a paraid acquiring multiple cores does not brick itself if ElasticScalingMVP feature is enabled +Description: Test that a paraid acquiring multiple cores does not brick itself if ElasticScalingMVP feature is enabled in genesis Network: ./0012-elastic-scaling-mvp.toml Creds: config @@ -15,14 +15,5 @@ alice: js-script ./0012-register-para.js return is 0 within 600 seconds validator: reports substrate_block_height{status="finalized"} is at least 35 within 100 seconds -# Parachain will now be stalled -validator: parachain 2000 block height is lower than 20 within 300 seconds - -# Enable the ElasticScalingMVP node feature. -alice: js-script ./0012-enable-node-feature.js with "1" return is 0 within 600 seconds - -# Wait two sessions for the config to be updated. -sleep 120 seconds - # Ensure parachain is now making progress. validator: parachain 2000 block height is at least 30 within 200 seconds diff --git a/polkadot/zombienet_tests/functional/0012-enable-node-feature.js b/polkadot/zombienet_tests/functional/0012-enable-node-feature.js deleted file mode 100644 index 4822e1f66447..000000000000 --- a/polkadot/zombienet_tests/functional/0012-enable-node-feature.js +++ /dev/null @@ -1,37 +0,0 @@ -async function run(nodeName, networkInfo, index) { - const { wsUri, userDefinedTypes } = networkInfo.nodesByName[nodeName]; - const api = await zombie.connect(wsUri, userDefinedTypes); - - await zombie.util.cryptoWaitReady(); - - // account to submit tx - const keyring = new zombie.Keyring({ type: "sr25519" }); - const alice = keyring.addFromUri("//Alice"); - - await new Promise(async (resolve, reject) => { - const unsub = await api.tx.sudo - .sudo(api.tx.configuration.setNodeFeature(Number(index), true)) - .signAndSend(alice, ({ status, isError }) => { - if (status.isInBlock) { - console.log( - `Transaction included at blockhash ${status.asInBlock}`, - ); - } else if (status.isFinalized) { - console.log( - `Transaction finalized at blockHash ${status.asFinalized}`, - ); - unsub(); - return resolve(); - } else if (isError) { - console.log(`Transaction error`); - reject(`Transaction error`); - } - }); - }); - - - - return 0; -} - -module.exports = { run }; diff --git a/polkadot/zombienet_tests/misc/0001-paritydb.toml b/polkadot/zombienet_tests/misc/0001-paritydb.toml index 399f848d3ac4..b3ce2081b111 100644 --- a/polkadot/zombienet_tests/misc/0001-paritydb.toml +++ b/polkadot/zombienet_tests/misc/0001-paritydb.toml @@ -3,9 +3,11 @@ timeout = 1000 bootnode = true [relaychain.genesis.runtimeGenesis.patch.configuration.config] - max_validators_per_core = 1 needed_approvals = 3 +[relaychain.genesis.runtimeGenesis.patch.configuration.config.scheduler_params] + max_validators_per_core = 1 + [relaychain] default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" chain = "rococo-local" diff --git a/prdoc/pr_1781.prdoc b/prdoc/pr_1781.prdoc new file mode 100644 index 000000000000..e3560842d15a --- /dev/null +++ b/prdoc/pr_1781.prdoc @@ -0,0 +1,45 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: "Multi-Block-Migrations, `poll` hook and new System Callbacks" + +doc: + - audience: Runtime Dev + description: | + The major things that this MR touches are: + + **Multi-Block-Migrations**: `pallet-migrations` is introduced that can be configured in the + `System` of a runtime to act as multi-block migrator. The `migrations` pallet then in turn + receives the list of MBMs as config parameter. The list of migrations can be an aggregated + tuple of `SteppedMigration` trait implementation. + It is paramount that the `migrations` pallet is configured in `System` once it is deployed. A + test is in place to double check this. + + To integrate this into your runtime, it is only necessary to change the return type of + `initialize_block` to `RuntimeExecutiveMode`. For extended info please see + https://github.com/paritytech/polkadot-sdk/pull/1781. + + **poll**: a new pallet hook named `poll` is added. This can be used for places where the code + that should be executed is not deadline critical. Runtime devs are advised to skim their usage + of `on_initialize` and `on_finalize` to see whether they can be replace with `poll`. `poll` is + not guaranteed to be called each block. In fact it will not be called when MBMs are ongoing. + + **System Callbacks**: The `system` pallet gets five new config items - all of which can be + safely set to `()` as default. They are: + - `SingleBlockMigrations`: replaces the `Executive` now for configuring migrations. + - `MultiBlockMigrator`: the `pallet-migrations` would be set here, if deployed. + - `PreInherents`: a hook that runs before any inherent. + - `PostInherents`: a hook to run between inherents and `poll`/MBM logic. + - `PostTransactions`: a hook to run after all transactions but before `on_idle`. + +crates: + - name: frame-executive + - name: frame-system + - name: frame-support + - name: frame-support-procedural + - name: pallet-migrations + - name: sc-basic-authorship + - name: sc-block-builder + - name: sp-api + - name: sp-api-proc-macro + - name: sp-runtime diff --git a/prdoc/pr_3187.prdoc b/prdoc/pr_3187.prdoc new file mode 100644 index 000000000000..bda41142d86c --- /dev/null +++ b/prdoc/pr_3187.prdoc @@ -0,0 +1,16 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: Retrying an execution on failed runtime construction + +doc: + - audience: Node Dev + description: | + If a runtime construction error happened during the execution request, then the artifact is re-prepared + and the execution request is retried at most once. See also the related issue. + +crates: + - name: polkadot-node-core-candidate-validation + - name: polkadot-node-core-pvf + - name: polkadot-node-core-pvf-execute-worker + - name: polkadot-node-core-pvf-common diff --git a/prdoc/pr_3491.prdoc b/prdoc/pr_3491.prdoc new file mode 100644 index 000000000000..e36afb916a62 --- /dev/null +++ b/prdoc/pr_3491.prdoc @@ -0,0 +1,12 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: Remove Deprecated OldWeight + +doc: + - audience: Runtime Dev + description: | + Removed deprecated sp_weights::OldWeight type. Use [`weight_v2::Weight`] instead. + +crates: + - name: frame-support diff --git a/prdoc/pr_3510.prdoc b/prdoc/pr_3510.prdoc new file mode 100644 index 000000000000..6ee2f9a81f75 --- /dev/null +++ b/prdoc/pr_3510.prdoc @@ -0,0 +1,13 @@ +title: "Fix multi-collator parachain transition to async backing" + +doc: + - audience: Node Operator + description: | + The dynamic Aura slot duration, introduced in PR#3211, didn't take the block import pipeline + into account. The result was the parachain backed by multiple collators not being able to + keep producing blocks after its runtime was upgraded to support async backing, requiring to + restart all the collator nodes. This change fixes the issue, introducing the dynamic Aura + slot duration into the block import pipeline. + +crates: + - name: "polkadot-parachain-bin" diff --git a/prdoc/pr_3513.prdoc b/prdoc/pr_3513.prdoc new file mode 100644 index 000000000000..e1f2afe280a5 --- /dev/null +++ b/prdoc/pr_3513.prdoc @@ -0,0 +1,18 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: Fix call enum's metadata regression + +doc: + - audience: Runtime Dev + - audience: Runtime User + description: | + This PR fixes an issue where in the metadata of all FRAME-based chains, the documentation for + all extrinsic/call/transactions has been replaced by a single line saying "see Pallet::name". + + Many wallets display the metadata of transactions to the user before signing, and this bug + might have affected this process. + +crates: + - name: frame + - name: frame-support diff --git a/prdoc/schema_user.json b/prdoc/schema_user.json index 82215d51866b..1bd0b3b93ee4 100644 --- a/prdoc/schema_user.json +++ b/prdoc/schema_user.json @@ -3,9 +3,8 @@ "$id": "https://raw.githubusercontent.com/paritytech/prdoc/master/prdoc_schema_user.json", "version": { "major": 1, - "minor": 0, - "patch": 0, - "timestamp": 20230817152351 + "minor": 1, + "patch": 0 }, "title": "Polkadot SDK PRDoc Schema", "description": "JSON Schema definition for the Polkadot SDK PR documentation", @@ -125,10 +124,16 @@ "name": { "type": "string" }, + "bump": { + "$ref": "#/$defs/semver_bump" + }, "note": { "type": "string" } - } + }, + "required": [ + "name" + ] }, "migration_db": { "type": "object", @@ -165,6 +170,26 @@ "description" ] }, + "semver_bump": { + "description": "The type of bump to apply to the crate version according to Cargo SemVer: https://doc.rust-lang.org/cargo/reference/semver.html. Please check docs/RELEASE.md for more information.", + "oneOf": [ + { + "const": "major", + "title": "Major", + "description": "A bump to the leftmost non-zero digit of the version number." + }, + { + "const": "minor", + "title": "Minor", + "description": "A bump to the second leftmost non-zero digit of the version number." + }, + { + "const": "patch", + "title": "Patch", + "description": "A bump to the third leftmost non-zero digit of the version number." + } + ] + }, "doc": { "type": "object", "description": "You have the the option to provide different description of your PR for different audiences.", diff --git a/substrate/bin/minimal/runtime/src/lib.rs b/substrate/bin/minimal/runtime/src/lib.rs index 610289693d91..fb996aef46b8 100644 --- a/substrate/bin/minimal/runtime/src/lib.rs +++ b/substrate/bin/minimal/runtime/src/lib.rs @@ -26,7 +26,8 @@ use frame::{ prelude::*, runtime::{ apis::{ - self, impl_runtime_apis, ApplyExtrinsicResult, CheckInherentsResult, OpaqueMetadata, + self, impl_runtime_apis, ApplyExtrinsicResult, CheckInherentsResult, + ExtrinsicInclusionMode, OpaqueMetadata, }, prelude::*, }, @@ -121,7 +122,7 @@ impl_runtime_apis! { RuntimeExecutive::execute_block(block) } - fn initialize_block(header: &Header) { + fn initialize_block(header: &Header) -> ExtrinsicInclusionMode { RuntimeExecutive::initialize_block(header) } } diff --git a/substrate/bin/node-template/node/src/service.rs b/substrate/bin/node-template/node/src/service.rs index 25cd65117841..125cca139279 100644 --- a/substrate/bin/node-template/node/src/service.rs +++ b/substrate/bin/node-template/node/src/service.rs @@ -80,23 +80,29 @@ pub fn new_partial(config: &Configuration) -> Result { telemetry.as_ref().map(|x| x.handle()), )?; - let slot_duration = sc_consensus_aura::slot_duration(&*client)?; - + let cidp_client = client.clone(); let import_queue = sc_consensus_aura::import_queue::(ImportQueueParams { block_import: grandpa_block_import.clone(), justification_import: Some(Box::new(grandpa_block_import.clone())), client: client.clone(), - create_inherent_data_providers: move |_, ()| async move { - let timestamp = sp_timestamp::InherentDataProvider::from_system_time(); + create_inherent_data_providers: move |parent_hash, _| { + let cidp_client = cidp_client.clone(); + async move { + let slot_duration = sc_consensus_aura::standalone::slot_duration_at( + &*cidp_client, + parent_hash, + )?; + let timestamp = sp_timestamp::InherentDataProvider::from_system_time(); - let slot = - sp_consensus_aura::inherents::InherentDataProvider::from_timestamp_and_slot_duration( - *timestamp, - slot_duration, - ); + let slot = + sp_consensus_aura::inherents::InherentDataProvider::from_timestamp_and_slot_duration( + *timestamp, + slot_duration, + ); - Ok((slot, timestamp)) + Ok((slot, timestamp)) + } }, spawner: &task_manager.spawn_essential_handle(), registry: config.prometheus_registry(), diff --git a/substrate/bin/node-template/runtime/src/lib.rs b/substrate/bin/node-template/runtime/src/lib.rs index 3b6a74be2512..159697f427f6 100644 --- a/substrate/bin/node-template/runtime/src/lib.rs +++ b/substrate/bin/node-template/runtime/src/lib.rs @@ -331,7 +331,7 @@ impl_runtime_apis! { Executive::execute_block(block); } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/substrate/bin/node/runtime/Cargo.toml b/substrate/bin/node/runtime/Cargo.toml index aa13c3d292be..9607d05daf0e 100644 --- a/substrate/bin/node/runtime/Cargo.toml +++ b/substrate/bin/node/runtime/Cargo.toml @@ -88,6 +88,7 @@ pallet-election-provider-support-benchmarking = { path = "../../../frame/electio pallet-elections-phragmen = { path = "../../../frame/elections-phragmen", default-features = false } pallet-example-tasks = { path = "../../../frame/examples/tasks", default-features = false } pallet-fast-unstake = { path = "../../../frame/fast-unstake", default-features = false } +pallet-migrations = { path = "../../../frame/migrations", default-features = false } pallet-nis = { path = "../../../frame/nis", default-features = false } pallet-grandpa = { path = "../../../frame/grandpa", default-features = false } pallet-im-online = { path = "../../../frame/im-online", default-features = false } @@ -198,6 +199,7 @@ std = [ "pallet-lottery/std", "pallet-membership/std", "pallet-message-queue/std", + "pallet-migrations/std", "pallet-mixnet/std", "pallet-mmr/std", "pallet-multisig/std", @@ -302,6 +304,7 @@ runtime-benchmarks = [ "pallet-lottery/runtime-benchmarks", "pallet-membership/runtime-benchmarks", "pallet-message-queue/runtime-benchmarks", + "pallet-migrations/runtime-benchmarks", "pallet-mixnet/runtime-benchmarks", "pallet-mmr/runtime-benchmarks", "pallet-multisig/runtime-benchmarks", @@ -381,6 +384,7 @@ try-runtime = [ "pallet-lottery/try-runtime", "pallet-membership/try-runtime", "pallet-message-queue/try-runtime", + "pallet-migrations/try-runtime", "pallet-mixnet/try-runtime", "pallet-mmr/try-runtime", "pallet-multisig/try-runtime", diff --git a/substrate/bin/node/runtime/src/lib.rs b/substrate/bin/node/runtime/src/lib.rs index b34d8c89e568..5431628c747d 100644 --- a/substrate/bin/node/runtime/src/lib.rs +++ b/substrate/bin/node/runtime/src/lib.rs @@ -310,6 +310,7 @@ impl frame_system::Config for Runtime { type SystemWeightInfo = frame_system::weights::SubstrateWeight; type SS58Prefix = ConstU16<42>; type MaxConsumers = ConstU32<16>; + type MultiBlockMigrator = MultiBlockMigrations; } impl pallet_insecure_randomness_collective_flip::Config for Runtime {} @@ -2007,6 +2008,25 @@ impl pallet_statement::Config for Runtime { type MaxAllowedBytes = MaxAllowedBytes; } +parameter_types! { + pub MbmServiceWeight: Weight = Perbill::from_percent(80) * RuntimeBlockWeights::get().max_block; +} + +impl pallet_migrations::Config for Runtime { + type RuntimeEvent = RuntimeEvent; + #[cfg(not(feature = "runtime-benchmarks"))] + type Migrations = (); + // Benchmarks need mocked migrations to guarantee that they succeed. + #[cfg(feature = "runtime-benchmarks")] + type Migrations = pallet_migrations::mock_helpers::MockedMigrations; + type CursorMaxLen = ConstU32<65_536>; + type IdentifierMaxLen = ConstU32<256>; + type MigrationStatusHandler = (); + type FailedMigrationHandler = frame_support::migrations::FreezeChainOnFailedMigration; + type MaxServiceWeight = MbmServiceWeight; + type WeightInfo = pallet_migrations::weights::SubstrateWeight; +} + parameter_types! { pub const BrokerPalletId: PalletId = PalletId(*b"py/broke"); } @@ -2241,6 +2261,7 @@ construct_runtime!( TxPause: pallet_tx_pause, SafeMode: pallet_safe_mode, Statement: pallet_statement, + MultiBlockMigrations: pallet_migrations, Broker: pallet_broker, TasksExample: pallet_example_tasks, Mixnet: pallet_mixnet, @@ -2372,6 +2393,7 @@ mod benches { [pallet_lottery, Lottery] [pallet_membership, TechnicalMembership] [pallet_message_queue, MessageQueue] + [pallet_migrations, MultiBlockMigrations] [pallet_mmr, Mmr] [pallet_multisig, Multisig] [pallet_nomination_pools, NominationPoolsBench::] @@ -2417,7 +2439,7 @@ impl_runtime_apis! { Executive::execute_block(block); } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/substrate/client/basic-authorship/src/basic_authorship.rs b/substrate/client/basic-authorship/src/basic_authorship.rs index fdc3d4918de3..932287ac8657 100644 --- a/substrate/client/basic-authorship/src/basic_authorship.rs +++ b/substrate/client/basic-authorship/src/basic_authorship.rs @@ -38,7 +38,7 @@ use sp_core::traits::SpawnNamed; use sp_inherents::InherentData; use sp_runtime::{ traits::{BlakeTwo256, Block as BlockT, Hash as HashT, Header as HeaderT}, - Digest, Percent, SaturatedConversion, + Digest, ExtrinsicInclusionMode, Percent, SaturatedConversion, }; use std::{marker::PhantomData, pin::Pin, sync::Arc, time}; @@ -335,11 +335,12 @@ where self.apply_inherents(&mut block_builder, inherent_data)?; - // TODO call `after_inherents` and check if we should apply extrinsincs here - // - - let end_reason = - self.apply_extrinsics(&mut block_builder, deadline, block_size_limit).await?; + let mode = block_builder.extrinsic_inclusion_mode(); + let end_reason = match mode { + ExtrinsicInclusionMode::AllExtrinsics => + self.apply_extrinsics(&mut block_builder, deadline, block_size_limit).await?, + ExtrinsicInclusionMode::OnlyInherents => EndProposingReason::TransactionForbidden, + }; let (block, storage_changes, proof) = block_builder.build()?.into_inner(); let block_took = block_timer.elapsed(); diff --git a/substrate/client/block-builder/src/lib.rs b/substrate/client/block-builder/src/lib.rs index 258e39d962b2..2f22cd42591f 100644 --- a/substrate/client/block-builder/src/lib.rs +++ b/substrate/client/block-builder/src/lib.rs @@ -37,7 +37,7 @@ use sp_core::traits::CallContext; use sp_runtime::{ legacy, traits::{Block as BlockT, Hash, HashingFor, Header as HeaderT, NumberFor, One}, - Digest, + Digest, ExtrinsicInclusionMode, }; use std::marker::PhantomData; @@ -198,10 +198,12 @@ pub struct BlockBuilder<'a, Block: BlockT, C: ProvideRuntimeApi + 'a> { extrinsics: Vec, api: ApiRef<'a, C::Api>, call_api_at: &'a C, + /// Version of the [`BlockBuilderApi`] runtime API. version: u32, parent_hash: Block::Hash, /// The estimated size of the block header. estimated_header_size: usize, + extrinsic_inclusion_mode: ExtrinsicInclusionMode, } impl<'a, Block, C> BlockBuilder<'a, Block, C> @@ -244,9 +246,19 @@ where api.set_call_context(CallContext::Onchain); - api.initialize_block(parent_hash, &header)?; + let core_version = api + .api_version::>(parent_hash)? + .ok_or_else(|| Error::VersionInvalid("Core".to_string()))?; - let version = api + let extrinsic_inclusion_mode = if core_version >= 5 { + api.initialize_block(parent_hash, &header)? + } else { + #[allow(deprecated)] + api.initialize_block_before_version_5(parent_hash, &header)?; + ExtrinsicInclusionMode::AllExtrinsics + }; + + let bb_version = api .api_version::>(parent_hash)? .ok_or_else(|| Error::VersionInvalid("BlockBuilderApi".to_string()))?; @@ -254,12 +266,18 @@ where parent_hash, extrinsics: Vec::new(), api, - version, + version: bb_version, estimated_header_size, call_api_at, + extrinsic_inclusion_mode, }) } + /// The extrinsic inclusion mode of the runtime for this block. + pub fn extrinsic_inclusion_mode(&self) -> ExtrinsicInclusionMode { + self.extrinsic_inclusion_mode + } + /// Push onto the block's list of extrinsics. /// /// This will ensure the extrinsic can be validly executed (by executing it). diff --git a/substrate/client/proposer-metrics/src/lib.rs b/substrate/client/proposer-metrics/src/lib.rs index 012e8ca769a9..2856300cf802 100644 --- a/substrate/client/proposer-metrics/src/lib.rs +++ b/substrate/client/proposer-metrics/src/lib.rs @@ -44,11 +44,14 @@ impl MetricsLink { } /// The reason why proposing a block ended. +#[derive(Clone, Copy, PartialEq, Eq)] pub enum EndProposingReason { NoMoreTransactions, HitDeadline, HitBlockSizeLimit, HitBlockWeightLimit, + /// No transactions are allowed in the block. + TransactionForbidden, } /// Authorship metrics. @@ -112,6 +115,7 @@ impl Metrics { EndProposingReason::NoMoreTransactions => "no_more_transactions", EndProposingReason::HitBlockSizeLimit => "hit_block_size_limit", EndProposingReason::HitBlockWeightLimit => "hit_block_weight_limit", + EndProposingReason::TransactionForbidden => "transactions_forbidden", }; self.end_proposing_reason.with_label_values(&[reason]).inc(); diff --git a/substrate/client/rpc-spec-v2/src/chain_head/test_utils.rs b/substrate/client/rpc-spec-v2/src/chain_head/test_utils.rs index d63a98a5cb0d..e81bd4bfa0b0 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/test_utils.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/test_utils.rs @@ -63,7 +63,7 @@ impl ChainHeadMockClient { BlockImportNotification::new(header.hash(), BlockOrigin::Own, header, true, None, sink); for sink in self.import_sinks.lock().iter_mut() { - sink.unbounded_send(notification.clone()).unwrap(); + let _ = sink.unbounded_send(notification.clone()); } } @@ -83,7 +83,7 @@ impl ChainHeadMockClient { let notification = FinalityNotification::from_summary(summary, sink); for sink in self.finality_sinks.lock().iter_mut() { - sink.unbounded_send(notification.clone()).unwrap(); + let _ = sink.unbounded_send(notification.clone()); } } } diff --git a/substrate/client/rpc-spec-v2/src/chain_head/tests.rs b/substrate/client/rpc-spec-v2/src/chain_head/tests.rs index 9544736d84c8..89d8c4ce2713 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/tests.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/tests.rs @@ -242,12 +242,12 @@ async fn follow_with_runtime() { let event: FollowEvent = get_next_event(&mut sub).await; // it is basically json-encoded substrate_test_runtime_client::runtime::VERSION - let runtime_str = "{\"specName\":\"test\",\"implName\":\"parity-test\",\"authoringVersion\":0,\ - \"specVersion\":2,\"implVersion\":2,\"apis\":[[\"0xdf6acb689907609b\",4],\ + let runtime_str = "{\"specName\":\"test\",\"implName\":\"parity-test\",\"authoringVersion\":1,\ + \"specVersion\":2,\"implVersion\":2,\"apis\":[[\"0xdf6acb689907609b\",5],\ [\"0x37e397fc7c91f5e4\",2],[\"0xd2bc9897eed08f15\",3],[\"0x40fe3ad401f8959a\",6],\ [\"0xbc9d89904f5b923f\",1],[\"0xc6e9a76309f39b09\",2],[\"0xdd718d5cc53262d4\",1],\ [\"0xcbca25e39f142387\",2],[\"0xf78b278be53f454c\",2],[\"0xab3c0572291feb8b\",1],\ - [\"0xed99c5acb25eedf5\",3],[\"0xfbc577b9d747efd6\",1]],\"transactionVersion\":1,\"stateVersion\":0}"; + [\"0xed99c5acb25eedf5\",3],[\"0xfbc577b9d747efd6\",1]],\"transactionVersion\":1,\"stateVersion\":1}"; let runtime: RuntimeVersion = serde_json::from_str(runtime_str).unwrap(); diff --git a/substrate/client/rpc-spec-v2/src/transaction/tests.rs b/substrate/client/rpc-spec-v2/src/transaction/tests.rs deleted file mode 100644 index 382f5adeae19..000000000000 --- a/substrate/client/rpc-spec-v2/src/transaction/tests.rs +++ /dev/null @@ -1,238 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -use super::*; -use crate::{ - chain_head::test_utils::ChainHeadMockClient, hex_string, - transaction::TransactionBroadcast as RpcTransactionBroadcast, -}; -use assert_matches::assert_matches; -use codec::Encode; -use futures::Future; -use jsonrpsee::{rpc_params, MethodsError as Error, RpcModule}; -use sc_transaction_pool::*; -use sc_transaction_pool_api::{ChainEvent, MaintainedTransactionPool, TransactionPool}; -use sp_core::{testing::TaskExecutor, traits::SpawnNamed}; -use std::{pin::Pin, sync::Arc, time::Duration}; -use substrate_test_runtime_client::{prelude::*, AccountKeyring::*, Client}; -use substrate_test_runtime_transaction_pool::{uxt, TestApi}; -use tokio::sync::mpsc; - -type Block = substrate_test_runtime_client::runtime::Block; - -/// Wrap the `TaskExecutor` to know when the broadcast future is dropped. -#[derive(Clone)] -struct TaskExecutorBroadcast { - executor: TaskExecutor, - sender: mpsc::UnboundedSender<()>, -} - -/// The channel that receives events when the broadcast futures are dropped. -type TaskExecutorRecv = mpsc::UnboundedReceiver<()>; - -impl TaskExecutorBroadcast { - /// Construct a new `TaskExecutorBroadcast` and a receiver to know when the broadcast futures - /// are dropped. - fn new() -> (Self, TaskExecutorRecv) { - let (sender, recv) = mpsc::unbounded_channel(); - - (Self { executor: TaskExecutor::new(), sender }, recv) - } -} - -impl SpawnNamed for TaskExecutorBroadcast { - fn spawn( - &self, - name: &'static str, - group: Option<&'static str>, - future: futures::future::BoxFuture<'static, ()>, - ) { - let sender = self.sender.clone(); - let future = Box::pin(async move { - future.await; - let _ = sender.send(()); - }); - - self.executor.spawn(name, group, future) - } - - fn spawn_blocking( - &self, - name: &'static str, - group: Option<&'static str>, - future: futures::future::BoxFuture<'static, ()>, - ) { - let sender = self.sender.clone(); - let future = Box::pin(async move { - future.await; - let _ = sender.send(()); - }); - - self.executor.spawn_blocking(name, group, future) - } -} - -/// Initial Alice account nonce. -const ALICE_NONCE: u64 = 209; - -fn create_basic_pool_with_genesis( - test_api: Arc, -) -> (BasicPool, Pin + Send>>) { - let genesis_hash = { - test_api - .chain() - .read() - .block_by_number - .get(&0) - .map(|blocks| blocks[0].0.header.hash()) - .expect("there is block 0. qed") - }; - BasicPool::new_test(test_api, genesis_hash, genesis_hash) -} - -fn maintained_pool() -> (BasicPool, Arc, futures::executor::ThreadPool) { - let api = Arc::new(TestApi::with_alice_nonce(ALICE_NONCE)); - let (pool, background_task) = create_basic_pool_with_genesis(api.clone()); - - let thread_pool = futures::executor::ThreadPool::new().unwrap(); - thread_pool.spawn_ok(background_task); - (pool, api, thread_pool) -} - -fn setup_api() -> ( - Arc, - Arc>, - Arc>>, - RpcModule< - TransactionBroadcast, ChainHeadMockClient>>, - >, - TaskExecutorRecv, -) { - let (pool, api, _) = maintained_pool(); - let pool = Arc::new(pool); - - let builder = TestClientBuilder::new(); - let client = Arc::new(builder.build()); - let client_mock = Arc::new(ChainHeadMockClient::new(client.clone())); - - let (task_executor, executor_recv) = TaskExecutorBroadcast::new(); - - let tx_api = - RpcTransactionBroadcast::new(client_mock.clone(), pool.clone(), Arc::new(task_executor)) - .into_rpc(); - - (api, pool, client_mock, tx_api, executor_recv) -} - -#[tokio::test] -async fn tx_broadcast_enters_pool() { - let (api, pool, client_mock, tx_api, _) = setup_api(); - - // Start at block 1. - let block_1_header = api.push_block(1, vec![], true); - - let uxt = uxt(Alice, ALICE_NONCE); - let xt = hex_string(&uxt.encode()); - - let operation_id: String = - tx_api.call("transaction_unstable_broadcast", rpc_params![&xt]).await.unwrap(); - - // Announce block 1 to `transaction_unstable_broadcast`. - client_mock.trigger_import_stream(block_1_header).await; - - // Ensure the tx propagated from `transaction_unstable_broadcast` to the transaction pool. - - // TODO: Improve testability by extending the `transaction_unstable_broadcast` with - // a middleware trait that intercepts the transaction status for testing. - let mut num_retries = 12; - while num_retries > 0 && pool.status().ready != 1 { - tokio::time::sleep(Duration::from_secs(5)).await; - num_retries -= 1; - } - assert_eq!(1, pool.status().ready); - assert_eq!(uxt.encode().len(), pool.status().ready_bytes); - - // Import block 2 with the transaction included. - let block_2_header = api.push_block(2, vec![uxt.clone()], true); - let block_2 = block_2_header.hash(); - - // Announce block 2 to the pool. - let event = ChainEvent::NewBestBlock { hash: block_2, tree_route: None }; - pool.maintain(event).await; - - assert_eq!(0, pool.status().ready); - - // Stop call can still be made. - let _: () = tx_api - .call("transaction_unstable_stop", rpc_params![&operation_id]) - .await - .unwrap(); -} - -#[tokio::test] -async fn tx_broadcast_invalid_tx() { - let (_, pool, _, tx_api, mut exec_recv) = setup_api(); - - // Invalid parameters. - let err = tx_api - .call::<_, serde_json::Value>("transaction_unstable_broadcast", [1u8]) - .await - .unwrap_err(); - assert_matches!(err, - Error::JsonRpc(err) if err.code() == super::error::json_rpc_spec::INVALID_PARAM_ERROR && err.message() == "Invalid params" - ); - - assert_eq!(0, pool.status().ready); - - // Invalid transaction that cannot be decoded. The broadcast silently exits. - let xt = "0xdeadbeef"; - let operation_id: String = - tx_api.call("transaction_unstable_broadcast", rpc_params![&xt]).await.unwrap(); - - assert_eq!(0, pool.status().ready); - - // Await the broadcast future to exit. - // Without this we'd be subject to races, where we try to call the stop before the tx is - // dropped. - exec_recv.recv().await.unwrap(); - - // The broadcast future was dropped, and the operation is no longer active. - // When the operation is not active, either from the tx being finalized or a - // terminal error; the stop method should return an error. - let err = tx_api - .call::<_, serde_json::Value>("transaction_unstable_stop", rpc_params![&operation_id]) - .await - .unwrap_err(); - assert_matches!(err, - Error::JsonRpc(err) if err.code() == super::error::json_rpc_spec::INVALID_PARAM_ERROR && err.message() == "Invalid operation id" - ); -} - -#[tokio::test] -async fn tx_invalid_stop() { - let (_, _, _, tx_api, _) = setup_api(); - - // Make an invalid stop call. - let err = tx_api - .call::<_, serde_json::Value>("transaction_unstable_stop", ["invalid_operation_id"]) - .await - .unwrap_err(); - assert_matches!(err, - Error::JsonRpc(err) if err.code() == super::error::json_rpc_spec::INVALID_PARAM_ERROR && err.message() == "Invalid operation id" - ); -} diff --git a/substrate/client/rpc-spec-v2/src/transaction/tests/executor.rs b/substrate/client/rpc-spec-v2/src/transaction/tests/executor.rs new file mode 100644 index 000000000000..ff9aca79887c --- /dev/null +++ b/substrate/client/rpc-spec-v2/src/transaction/tests/executor.rs @@ -0,0 +1,100 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use sp_core::{testing::TaskExecutor, traits::SpawnNamed}; +use std::sync::{atomic::AtomicUsize, Arc}; +use tokio::sync::mpsc; + +/// Wrap the `TaskExecutor` to know when the broadcast future is dropped. +#[derive(Clone)] +pub struct TaskExecutorBroadcast { + executor: TaskExecutor, + sender: mpsc::UnboundedSender<()>, + num_tasks: Arc, +} + +/// The channel that receives events when the broadcast futures are dropped. +pub type TaskExecutorRecv = mpsc::UnboundedReceiver<()>; + +/// The state of the `TaskExecutorBroadcast`. +pub struct TaskExecutorState { + pub recv: TaskExecutorRecv, + pub num_tasks: Arc, +} + +impl TaskExecutorState { + pub fn num_tasks(&self) -> usize { + self.num_tasks.load(std::sync::atomic::Ordering::Acquire) + } +} + +impl TaskExecutorBroadcast { + /// Construct a new `TaskExecutorBroadcast` and a receiver to know when the broadcast futures + /// are dropped. + pub fn new() -> (Self, TaskExecutorState) { + let (sender, recv) = mpsc::unbounded_channel(); + let num_tasks = Arc::new(AtomicUsize::new(0)); + + ( + Self { executor: TaskExecutor::new(), sender, num_tasks: num_tasks.clone() }, + TaskExecutorState { recv, num_tasks }, + ) + } +} + +impl SpawnNamed for TaskExecutorBroadcast { + fn spawn( + &self, + name: &'static str, + group: Option<&'static str>, + future: futures::future::BoxFuture<'static, ()>, + ) { + let sender = self.sender.clone(); + let num_tasks = self.num_tasks.clone(); + + let future = Box::pin(async move { + num_tasks.fetch_add(1, std::sync::atomic::Ordering::AcqRel); + future.await; + num_tasks.fetch_sub(1, std::sync::atomic::Ordering::AcqRel); + + let _ = sender.send(()); + }); + + self.executor.spawn(name, group, future) + } + + fn spawn_blocking( + &self, + name: &'static str, + group: Option<&'static str>, + future: futures::future::BoxFuture<'static, ()>, + ) { + let sender = self.sender.clone(); + let num_tasks = self.num_tasks.clone(); + + let future = Box::pin(async move { + num_tasks.fetch_add(1, std::sync::atomic::Ordering::AcqRel); + future.await; + num_tasks.fetch_sub(1, std::sync::atomic::Ordering::AcqRel); + + let _ = sender.send(()); + }); + + self.executor.spawn_blocking(name, group, future) + } +} diff --git a/substrate/client/rpc-spec-v2/src/transaction/tests/middleware_pool.rs b/substrate/client/rpc-spec-v2/src/transaction/tests/middleware_pool.rs new file mode 100644 index 000000000000..aa8ac572dec9 --- /dev/null +++ b/substrate/client/rpc-spec-v2/src/transaction/tests/middleware_pool.rs @@ -0,0 +1,187 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use codec::Encode; +use futures::Future; +use sc_transaction_pool::BasicPool; +use sc_transaction_pool_api::{ + ImportNotificationStream, PoolFuture, PoolStatus, ReadyTransactions, TransactionFor, + TransactionPool, TransactionSource, TransactionStatusStreamFor, TxHash, +}; + +use crate::hex_string; +use futures::{FutureExt, StreamExt}; + +use sp_runtime::traits::{Block as BlockT, NumberFor}; +use std::{collections::HashMap, pin::Pin, sync::Arc}; +use substrate_test_runtime_transaction_pool::TestApi; +use tokio::sync::mpsc; + +pub type Block = substrate_test_runtime_client::runtime::Block; + +pub type TxTestPool = MiddlewarePool; +pub type TxStatusType = sc_transaction_pool_api::TransactionStatus< + sc_transaction_pool_api::TxHash, + sc_transaction_pool_api::BlockHash, +>; +pub type TxStatusTypeTest = TxStatusType; + +/// The type of the event that the middleware captures. +#[derive(Debug, PartialEq)] +pub enum MiddlewarePoolEvent { + TransactionStatus { + transaction: String, + status: sc_transaction_pool_api::TransactionStatus< + ::Hash, + ::Hash, + >, + }, + PoolError { + transaction: String, + err: String, + }, +} + +/// The channel that receives events when the broadcast futures are dropped. +pub type MiddlewarePoolRecv = mpsc::UnboundedReceiver; + +/// Add a middleware to the transaction pool. +/// +/// This wraps the `submit_and_watch` to gain access to the events. +pub struct MiddlewarePool { + pub inner_pool: Arc>, + /// Send the middleware events to the test. + sender: mpsc::UnboundedSender, +} + +impl MiddlewarePool { + /// Construct a new [`MiddlewarePool`]. + pub fn new(pool: Arc>) -> (Self, MiddlewarePoolRecv) { + let (sender, recv) = mpsc::unbounded_channel(); + (MiddlewarePool { inner_pool: pool, sender }, recv) + } +} + +impl TransactionPool for MiddlewarePool { + type Block = as TransactionPool>::Block; + type Hash = as TransactionPool>::Hash; + type InPoolTransaction = as TransactionPool>::InPoolTransaction; + type Error = as TransactionPool>::Error; + + fn submit_at( + &self, + at: ::Hash, + source: TransactionSource, + xts: Vec>, + ) -> PoolFuture, Self::Error>>, Self::Error> { + self.inner_pool.submit_at(at, source, xts) + } + + fn submit_one( + &self, + at: ::Hash, + source: TransactionSource, + xt: TransactionFor, + ) -> PoolFuture, Self::Error> { + self.inner_pool.submit_one(at, source, xt) + } + + fn submit_and_watch( + &self, + at: ::Hash, + source: TransactionSource, + xt: TransactionFor, + ) -> PoolFuture>>, Self::Error> { + let pool = self.inner_pool.clone(); + let sender = self.sender.clone(); + let transaction = hex_string(&xt.encode()); + + async move { + let watcher = match pool.submit_and_watch(at, source, xt).await { + Ok(watcher) => watcher, + Err(err) => { + let _ = sender.send(MiddlewarePoolEvent::PoolError { + transaction: transaction.clone(), + err: err.to_string(), + }); + return Err(err); + }, + }; + + let watcher = watcher.map(move |status| { + let sender = sender.clone(); + let transaction = transaction.clone(); + + let _ = sender.send(MiddlewarePoolEvent::TransactionStatus { + transaction, + status: status.clone(), + }); + + status + }); + + Ok(watcher.boxed()) + } + .boxed() + } + + fn remove_invalid(&self, hashes: &[TxHash]) -> Vec> { + self.inner_pool.remove_invalid(hashes) + } + + fn status(&self) -> PoolStatus { + self.inner_pool.status() + } + + fn import_notification_stream(&self) -> ImportNotificationStream> { + self.inner_pool.import_notification_stream() + } + + fn hash_of(&self, xt: &TransactionFor) -> TxHash { + self.inner_pool.hash_of(xt) + } + + fn on_broadcasted(&self, propagations: HashMap, Vec>) { + self.inner_pool.on_broadcasted(propagations) + } + + fn ready_transaction(&self, hash: &TxHash) -> Option> { + self.inner_pool.ready_transaction(hash) + } + + fn ready_at( + &self, + at: NumberFor, + ) -> Pin< + Box< + dyn Future< + Output = Box> + Send>, + > + Send, + >, + > { + self.inner_pool.ready_at(at) + } + + fn ready(&self) -> Box> + Send> { + self.inner_pool.ready() + } + + fn futures(&self) -> Vec { + self.inner_pool.futures() + } +} diff --git a/substrate/client/rpc-spec-v2/src/transaction/tests/mod.rs b/substrate/client/rpc-spec-v2/src/transaction/tests/mod.rs new file mode 100644 index 000000000000..ab0caaf906fd --- /dev/null +++ b/substrate/client/rpc-spec-v2/src/transaction/tests/mod.rs @@ -0,0 +1,24 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +mod executor; +mod middleware_pool; +#[macro_use] +mod setup; + +mod transaction_broadcast_tests; diff --git a/substrate/client/rpc-spec-v2/src/transaction/tests/setup.rs b/substrate/client/rpc-spec-v2/src/transaction/tests/setup.rs new file mode 100644 index 000000000000..04ee7b9b4c94 --- /dev/null +++ b/substrate/client/rpc-spec-v2/src/transaction/tests/setup.rs @@ -0,0 +1,120 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use crate::{ + chain_head::test_utils::ChainHeadMockClient, + transaction::{ + api::TransactionBroadcastApiServer, + tests::executor::{TaskExecutorBroadcast, TaskExecutorState}, + TransactionBroadcast as RpcTransactionBroadcast, + }, +}; +use futures::Future; +use jsonrpsee::RpcModule; +use sc_transaction_pool::*; +use std::{pin::Pin, sync::Arc}; +use substrate_test_runtime_client::{prelude::*, Client}; +use substrate_test_runtime_transaction_pool::TestApi; + +use crate::transaction::tests::middleware_pool::{MiddlewarePool, MiddlewarePoolRecv}; + +pub type Block = substrate_test_runtime_client::runtime::Block; + +/// Initial Alice account nonce. +pub const ALICE_NONCE: u64 = 209; + +fn create_basic_pool_with_genesis( + test_api: Arc, + options: Options, +) -> (BasicPool, Pin + Send>>) { + let genesis_hash = { + test_api + .chain() + .read() + .block_by_number + .get(&0) + .map(|blocks| blocks[0].0.header.hash()) + .expect("there is block 0. qed") + }; + BasicPool::new_test(test_api, genesis_hash, genesis_hash, options) +} + +fn maintained_pool( + options: Options, +) -> (BasicPool, Arc, futures::executor::ThreadPool) { + let api = Arc::new(TestApi::with_alice_nonce(ALICE_NONCE)); + let (pool, background_task) = create_basic_pool_with_genesis(api.clone(), options); + + let thread_pool = futures::executor::ThreadPool::new().unwrap(); + thread_pool.spawn_ok(background_task); + (pool, api, thread_pool) +} + +pub fn setup_api( + options: Options, +) -> ( + Arc, + Arc, + Arc>>, + RpcModule>>>, + TaskExecutorState, + MiddlewarePoolRecv, +) { + let (pool, api, _) = maintained_pool(options); + let (pool, pool_state) = MiddlewarePool::new(Arc::new(pool).clone()); + let pool = Arc::new(pool); + + let builder = TestClientBuilder::new(); + let client = Arc::new(builder.build()); + let client_mock = Arc::new(ChainHeadMockClient::new(client.clone())); + + let (task_executor, executor_recv) = TaskExecutorBroadcast::new(); + + let tx_api = + RpcTransactionBroadcast::new(client_mock.clone(), pool.clone(), Arc::new(task_executor)) + .into_rpc(); + + (api, pool, client_mock, tx_api, executor_recv, pool_state) +} + +/// Get the next event from the provided middleware in at most 5 seconds. +macro_rules! get_next_event { + ($middleware:expr) => { + tokio::time::timeout(std::time::Duration::from_secs(5), $middleware.recv()) + .await + .unwrap() + .unwrap() + }; +} + +/// Collect the next number of transaction events from the provided middleware. +macro_rules! get_next_tx_events { + ($middleware:expr, $num:expr) => {{ + let mut events = std::collections::HashMap::new(); + for _ in 0..$num { + let event = get_next_event!($middleware); + match event { + crate::transaction::tests::middleware_pool::MiddlewarePoolEvent::TransactionStatus { transaction, status } => { + events.entry(transaction).or_insert_with(|| vec![]).push(status); + }, + other => panic!("Expected TransactionStatus, received {:?}", other), + }; + } + events + }}; +} diff --git a/substrate/client/rpc-spec-v2/src/transaction/tests/transaction_broadcast_tests.rs b/substrate/client/rpc-spec-v2/src/transaction/tests/transaction_broadcast_tests.rs new file mode 100644 index 000000000000..690a1a64d746 --- /dev/null +++ b/substrate/client/rpc-spec-v2/src/transaction/tests/transaction_broadcast_tests.rs @@ -0,0 +1,523 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use crate::{hex_string, transaction::error::json_rpc_spec}; +use assert_matches::assert_matches; +use codec::Encode; +use jsonrpsee::{rpc_params, MethodsError as Error}; +use sc_transaction_pool::{Options, PoolLimit}; +use sc_transaction_pool_api::{ChainEvent, MaintainedTransactionPool, TransactionPool}; +use std::sync::Arc; +use substrate_test_runtime_client::AccountKeyring::*; +use substrate_test_runtime_transaction_pool::uxt; + +// Test helpers. +use crate::transaction::tests::{ + middleware_pool::{MiddlewarePoolEvent, TxStatusTypeTest}, + setup::{setup_api, ALICE_NONCE}, +}; + +#[tokio::test] +async fn tx_broadcast_enters_pool() { + let (api, pool, client_mock, tx_api, mut exec_middleware, mut pool_middleware) = + setup_api(Default::default()); + + // Start at block 1. + let block_1_header = api.push_block(1, vec![], true); + + let uxt = uxt(Alice, ALICE_NONCE); + let xt = hex_string(&uxt.encode()); + + let operation_id: String = + tx_api.call("transaction_unstable_broadcast", rpc_params![&xt]).await.unwrap(); + + // Announce block 1 to `transaction_unstable_broadcast`. + client_mock.trigger_import_stream(block_1_header).await; + + // Ensure the tx propagated from `transaction_unstable_broadcast` to the transaction pool. + let event = get_next_event!(&mut pool_middleware); + assert_eq!( + event, + MiddlewarePoolEvent::TransactionStatus { + transaction: xt.clone(), + status: TxStatusTypeTest::Ready + } + ); + + assert_eq!(1, pool.inner_pool.status().ready); + assert_eq!(uxt.encode().len(), pool.inner_pool.status().ready_bytes); + + // Import block 2 with the transaction included. + let block_2_header = api.push_block(2, vec![uxt.clone()], true); + let block_2 = block_2_header.hash(); + + // Announce block 2 to the pool. + let event = ChainEvent::NewBestBlock { hash: block_2, tree_route: None }; + pool.inner_pool.maintain(event).await; + assert_eq!(0, pool.inner_pool.status().ready); + + let event = get_next_event!(&mut pool_middleware); + assert_eq!( + event, + MiddlewarePoolEvent::TransactionStatus { + transaction: xt.clone(), + status: TxStatusTypeTest::InBlock((block_2, 0)) + } + ); + + // The future broadcast awaits for the finalized status to be reached. + // Force the future to exit by calling stop. + let _: () = tx_api + .call("transaction_unstable_stop", rpc_params![&operation_id]) + .await + .unwrap(); + + // Ensure the broadcast future finishes. + let _ = get_next_event!(&mut exec_middleware.recv); + assert_eq!(0, exec_middleware.num_tasks()); +} + +#[tokio::test] +async fn tx_broadcast_invalid_tx() { + let (_, pool, _, tx_api, mut exec_middleware, _) = setup_api(Default::default()); + + // Invalid parameters. + let err = tx_api + .call::<_, serde_json::Value>("transaction_unstable_broadcast", [1u8]) + .await + .unwrap_err(); + assert_matches!(err, + Error::JsonRpc(err) if err.code() == json_rpc_spec::INVALID_PARAM_ERROR && err.message() == "Invalid params" + ); + + assert_eq!(0, pool.status().ready); + + // Invalid transaction that cannot be decoded. The broadcast silently exits. + let xt = "0xdeadbeef"; + let operation_id: String = + tx_api.call("transaction_unstable_broadcast", rpc_params![&xt]).await.unwrap(); + + assert_eq!(0, pool.status().ready); + + // Await the broadcast future to exit. + // Without this we'd be subject to races, where we try to call the stop before the tx is + // dropped. + let _ = get_next_event!(&mut exec_middleware.recv); + assert_eq!(0, exec_middleware.num_tasks()); + + // The broadcast future was dropped, and the operation is no longer active. + // When the operation is not active, either from the tx being finalized or a + // terminal error; the stop method should return an error. + let err = tx_api + .call::<_, serde_json::Value>("transaction_unstable_stop", rpc_params![&operation_id]) + .await + .unwrap_err(); + assert_matches!(err, + Error::JsonRpc(err) if err.code() == json_rpc_spec::INVALID_PARAM_ERROR && err.message() == "Invalid operation id" + ); +} + +#[tokio::test] +async fn tx_stop_with_invalid_operation_id() { + let (_, _, _, tx_api, _, _) = setup_api(Default::default()); + + // Make an invalid stop call. + let err = tx_api + .call::<_, serde_json::Value>("transaction_unstable_stop", ["invalid_operation_id"]) + .await + .unwrap_err(); + assert_matches!(err, + Error::JsonRpc(err) if err.code() == json_rpc_spec::INVALID_PARAM_ERROR && err.message() == "Invalid operation id" + ); +} + +#[tokio::test] +async fn tx_broadcast_resubmits_future_nonce_tx() { + let (api, pool, client_mock, tx_api, mut exec_middleware, mut pool_middleware) = + setup_api(Default::default()); + + // Start at block 1. + let block_1_header = api.push_block(1, vec![], true); + let block_1 = block_1_header.hash(); + + let current_uxt = uxt(Alice, ALICE_NONCE); + let current_xt = hex_string(¤t_uxt.encode()); + // This lives in the future. + let future_uxt = uxt(Alice, ALICE_NONCE + 1); + let future_xt = hex_string(&future_uxt.encode()); + + let future_operation_id: String = tx_api + .call("transaction_unstable_broadcast", rpc_params![&future_xt]) + .await + .unwrap(); + + // Announce block 1 to `transaction_unstable_broadcast`. + client_mock.trigger_import_stream(block_1_header).await; + + // Ensure the tx propagated from `transaction_unstable_broadcast` to the transaction pool. + let event = get_next_event!(&mut pool_middleware); + assert_eq!( + event, + MiddlewarePoolEvent::TransactionStatus { + transaction: future_xt.clone(), + status: TxStatusTypeTest::Future + } + ); + + let event = ChainEvent::NewBestBlock { hash: block_1, tree_route: None }; + pool.inner_pool.maintain(event).await; + assert_eq!(0, pool.inner_pool.status().ready); + // Ensure the tx is in the future. + assert_eq!(1, pool.inner_pool.status().future); + + let block_2_header = api.push_block(2, vec![], true); + let block_2 = block_2_header.hash(); + + let operation_id: String = tx_api + .call("transaction_unstable_broadcast", rpc_params![¤t_xt]) + .await + .unwrap(); + assert_ne!(future_operation_id, operation_id); + + // Announce block 2 to `transaction_unstable_broadcast`. + client_mock.trigger_import_stream(block_2_header).await; + + // Collect the events of both transactions. + let events = get_next_tx_events!(&mut pool_middleware, 2); + // Transactions entered the ready queue. + assert_eq!(events.get(¤t_xt).unwrap(), &vec![TxStatusTypeTest::Ready]); + assert_eq!(events.get(&future_xt).unwrap(), &vec![TxStatusTypeTest::Ready]); + + let event = ChainEvent::NewBestBlock { hash: block_2, tree_route: None }; + pool.inner_pool.maintain(event).await; + assert_eq!(2, pool.inner_pool.status().ready); + assert_eq!(0, pool.inner_pool.status().future); + + // Finalize transactions. + let block_3_header = api.push_block(3, vec![current_uxt, future_uxt], true); + let block_3 = block_3_header.hash(); + client_mock.trigger_import_stream(block_3_header).await; + + let event = ChainEvent::Finalized { hash: block_3, tree_route: Arc::from(vec![]) }; + pool.inner_pool.maintain(event).await; + assert_eq!(0, pool.inner_pool.status().ready); + assert_eq!(0, pool.inner_pool.status().future); + + let events = get_next_tx_events!(&mut pool_middleware, 4); + assert_eq!( + events.get(¤t_xt).unwrap(), + &vec![TxStatusTypeTest::InBlock((block_3, 0)), TxStatusTypeTest::Finalized((block_3, 0))] + ); + assert_eq!( + events.get(&future_xt).unwrap(), + &vec![TxStatusTypeTest::InBlock((block_3, 1)), TxStatusTypeTest::Finalized((block_3, 1))] + ); + + // Both broadcast futures must exit. + let _ = get_next_event!(&mut exec_middleware.recv); + let _ = get_next_event!(&mut exec_middleware.recv); + assert_eq!(0, exec_middleware.num_tasks()); +} + +/// This test is similar to `tx_broadcast_enters_pool` +/// However the last block is announced as finalized to force the +/// broadcast future to exit before the `stop` is called. +#[tokio::test] +async fn tx_broadcast_stop_after_broadcast_finishes() { + let (api, pool, client_mock, tx_api, mut exec_middleware, mut pool_middleware) = + setup_api(Default::default()); + + // Start at block 1. + let block_1_header = api.push_block(1, vec![], true); + + let uxt = uxt(Alice, ALICE_NONCE); + let xt = hex_string(&uxt.encode()); + + let operation_id: String = + tx_api.call("transaction_unstable_broadcast", rpc_params![&xt]).await.unwrap(); + + // Announce block 1 to `transaction_unstable_broadcast`. + client_mock.trigger_import_stream(block_1_header).await; + + // Ensure the tx propagated from `transaction_unstable_broadcast` to the transaction + // pool.inner_pool. + let event = get_next_event!(&mut pool_middleware); + assert_eq!( + event, + MiddlewarePoolEvent::TransactionStatus { + transaction: xt.clone(), + status: TxStatusTypeTest::Ready + } + ); + + assert_eq!(1, pool.inner_pool.status().ready); + assert_eq!(uxt.encode().len(), pool.inner_pool.status().ready_bytes); + + // Import block 2 with the transaction included. + let block_2_header = api.push_block(2, vec![uxt.clone()], true); + let block_2 = block_2_header.hash(); + + // Announce block 2 to the pool.inner_pool. + let event = ChainEvent::Finalized { hash: block_2, tree_route: Arc::from(vec![]) }; + pool.inner_pool.maintain(event).await; + + assert_eq!(0, pool.inner_pool.status().ready); + + let event = get_next_event!(&mut pool_middleware); + assert_eq!( + event, + MiddlewarePoolEvent::TransactionStatus { + transaction: xt.clone(), + status: TxStatusTypeTest::InBlock((block_2, 0)) + } + ); + + let event = get_next_event!(&mut pool_middleware); + assert_eq!( + event, + MiddlewarePoolEvent::TransactionStatus { + transaction: xt.clone(), + status: TxStatusTypeTest::Finalized((block_2, 0)) + } + ); + + // Ensure the broadcast future terminated properly. + let _ = get_next_event!(&mut exec_middleware.recv); + assert_eq!(0, exec_middleware.num_tasks()); + + // The operation ID is no longer valid, check that the broadcast future + // cleared out the inner state of the operation. + let err = tx_api + .call::<_, serde_json::Value>("transaction_unstable_stop", rpc_params![&operation_id]) + .await + .unwrap_err(); + assert_matches!(err, + Error::JsonRpc(err) if err.code() == json_rpc_spec::INVALID_PARAM_ERROR && err.message() == "Invalid operation id" + ); +} + +#[tokio::test] +async fn tx_broadcast_resubmits_invalid_tx() { + let limits = PoolLimit { count: 8192, total_bytes: 20 * 1024 * 1024 }; + let options = Options { + ready: limits.clone(), + future: limits, + reject_future_transactions: false, + // This ensures that a transaction is not banned. + ban_time: std::time::Duration::ZERO, + }; + + let (api, pool, client_mock, tx_api, mut exec_middleware, mut pool_middleware) = + setup_api(options); + + let uxt = uxt(Alice, ALICE_NONCE); + let xt = hex_string(&uxt.encode()); + let _operation_id: String = + tx_api.call("transaction_unstable_broadcast", rpc_params![&xt]).await.unwrap(); + + let block_1_header = api.push_block(1, vec![], true); + let block_1 = block_1_header.hash(); + // Announce block 1 to `transaction_unstable_broadcast`. + client_mock.trigger_import_stream(block_1_header).await; + + // Ensure the tx propagated from `transaction_unstable_broadcast` to the transaction pool. + let event = get_next_event!(&mut pool_middleware); + assert_eq!( + event, + MiddlewarePoolEvent::TransactionStatus { + transaction: xt.clone(), + status: TxStatusTypeTest::Ready, + } + ); + assert_eq!(1, pool.inner_pool.status().ready); + assert_eq!(uxt.encode().len(), pool.inner_pool.status().ready_bytes); + + // Mark the transaction as invalid from the API, causing a temporary ban. + api.add_invalid(&uxt); + + // Push an event to the pool to ensure the transaction is excluded. + let event = ChainEvent::NewBestBlock { hash: block_1, tree_route: None }; + pool.inner_pool.maintain(event).await; + assert_eq!(1, pool.inner_pool.status().ready); + + // Ensure the `transaction_unstable_broadcast` is aware of the invalid transaction. + let event = get_next_event!(&mut pool_middleware); + // Because we have received an `Invalid` status, we try to broadcast the transaction with the + // next announced block. + assert_eq!( + event, + MiddlewarePoolEvent::TransactionStatus { + transaction: xt.clone(), + status: TxStatusTypeTest::Invalid + } + ); + + // Import block 2. + let block_2_header = api.push_block(2, vec![], true); + client_mock.trigger_import_stream(block_2_header).await; + + // Ensure we propagate the temporary ban error to `submit_and_watch`. + // This ensures we'll loop again with the next annmounced block and try to resubmit the + // transaction. The transaction remains temporarily banned until the pool is maintained. + let event = get_next_event!(&mut pool_middleware); + assert_matches!(event, MiddlewarePoolEvent::PoolError { transaction, err } if transaction == xt && err.contains("Transaction temporarily Banned")); + + // Import block 3. + let block_3_header = api.push_block(3, vec![], true); + let block_3 = block_3_header.hash(); + // Remove the invalid transaction from the pool to allow it to pass through. + api.remove_invalid(&uxt); + let event = ChainEvent::NewBestBlock { hash: block_3, tree_route: None }; + // We have to maintain the pool to ensure the transaction is no longer invalid. + // This clears out the banned transactions. + pool.inner_pool.maintain(event).await; + assert_eq!(0, pool.inner_pool.status().ready); + + // Announce block to `transaction_unstable_broadcast`. + client_mock.trigger_import_stream(block_3_header).await; + + let event = get_next_event!(&mut pool_middleware); + assert_eq!( + event, + MiddlewarePoolEvent::TransactionStatus { + transaction: xt.clone(), + status: TxStatusTypeTest::Ready, + } + ); + assert_eq!(1, pool.inner_pool.status().ready); + + let block_4_header = api.push_block(4, vec![uxt], true); + let block_4 = block_4_header.hash(); + let event = ChainEvent::Finalized { hash: block_4, tree_route: Arc::from(vec![]) }; + pool.inner_pool.maintain(event).await; + + let event = get_next_event!(&mut pool_middleware); + assert_eq!( + event, + MiddlewarePoolEvent::TransactionStatus { + transaction: xt.clone(), + status: TxStatusTypeTest::InBlock((block_4, 0)), + } + ); + let event = get_next_event!(&mut pool_middleware); + assert_eq!( + event, + MiddlewarePoolEvent::TransactionStatus { + transaction: xt.clone(), + status: TxStatusTypeTest::Finalized((block_4, 0)), + } + ); + + // Ensure the broadcast future terminated properly. + let _ = get_next_event!(&mut exec_middleware.recv); + assert_eq!(0, exec_middleware.num_tasks()); +} + +/// This is similar to `tx_broadcast_resubmits_invalid_tx`. +/// However, it forces the tx to be resubmited because of the pool +/// limits. Which is a different code path than the invalid tx. +#[tokio::test] +async fn tx_broadcast_resubmits_dropped_tx() { + let limits = PoolLimit { count: 1, total_bytes: 1000 }; + let options = Options { + ready: limits.clone(), + future: limits, + reject_future_transactions: false, + // This ensures that a transaction is not banned. + ban_time: std::time::Duration::ZERO, + }; + + let (api, pool, client_mock, tx_api, _, mut pool_middleware) = setup_api(options); + + let current_uxt = uxt(Alice, ALICE_NONCE); + let current_xt = hex_string(¤t_uxt.encode()); + // This lives in the future. + let future_uxt = uxt(Alice, ALICE_NONCE + 1); + let future_xt = hex_string(&future_uxt.encode()); + + // By default the `validate_transaction` mock uses priority 1 for + // transactions. Bump the priority to ensure other transactions + // are immediately dropped. + api.set_priority(¤t_uxt, 10); + + let current_operation_id: String = tx_api + .call("transaction_unstable_broadcast", rpc_params![¤t_xt]) + .await + .unwrap(); + + // Announce block 1 to `transaction_unstable_broadcast`. + let block_1_header = api.push_block(1, vec![], true); + let event = + ChainEvent::Finalized { hash: block_1_header.hash(), tree_route: Arc::from(vec![]) }; + pool.inner_pool.maintain(event).await; + client_mock.trigger_import_stream(block_1_header).await; + + let event = get_next_event!(&mut pool_middleware); + assert_eq!( + event, + MiddlewarePoolEvent::TransactionStatus { + transaction: current_xt.clone(), + status: TxStatusTypeTest::Ready, + } + ); + assert_eq!(1, pool.inner_pool.status().ready); + + // The future tx has priority 2, smaller than the current 10. + api.set_priority(&future_uxt, 2); + let future_operation_id: String = tx_api + .call("transaction_unstable_broadcast", rpc_params![&future_xt]) + .await + .unwrap(); + assert_ne!(current_operation_id, future_operation_id); + + let block_2_header = api.push_block(2, vec![], true); + let event = + ChainEvent::Finalized { hash: block_2_header.hash(), tree_route: Arc::from(vec![]) }; + pool.inner_pool.maintain(event).await; + client_mock.trigger_import_stream(block_2_header).await; + + // We must have at most 1 transaction in the pool, as per limits above. + assert_eq!(1, pool.inner_pool.status().ready); + + let event = get_next_event!(&mut pool_middleware); + assert_eq!( + event, + MiddlewarePoolEvent::PoolError { + transaction: future_xt.clone(), + err: "Transaction couldn't enter the pool because of the limit".into() + } + ); + + let block_3_header = api.push_block(3, vec![current_uxt], true); + let event = + ChainEvent::Finalized { hash: block_3_header.hash(), tree_route: Arc::from(vec![]) }; + pool.inner_pool.maintain(event).await; + client_mock.trigger_import_stream(block_3_header.clone()).await; + + // The first tx is in a finalzied block; the future tx must enter the pool. + let events = get_next_tx_events!(&mut pool_middleware, 3); + assert_eq!( + events.get(¤t_xt).unwrap(), + &vec![ + TxStatusTypeTest::InBlock((block_3_header.hash(), 0)), + TxStatusTypeTest::Finalized((block_3_header.hash(), 0)) + ] + ); + // The dropped transaction was resubmitted. + assert_eq!(events.get(&future_xt).unwrap(), &vec![TxStatusTypeTest::Ready]); +} diff --git a/substrate/client/rpc/src/state/tests.rs b/substrate/client/rpc/src/state/tests.rs index 96f4c1be960f..dd866e671c50 100644 --- a/substrate/client/rpc/src/state/tests.rs +++ b/substrate/client/rpc/src/state/tests.rs @@ -475,7 +475,7 @@ async fn should_return_runtime_version() { // it is basically json-encoded substrate_test_runtime_client::runtime::VERSION let result = "{\"specName\":\"test\",\"implName\":\"parity-test\",\"authoringVersion\":1,\ - \"specVersion\":2,\"implVersion\":2,\"apis\":[[\"0xdf6acb689907609b\",4],\ + \"specVersion\":2,\"implVersion\":2,\"apis\":[[\"0xdf6acb689907609b\",5],\ [\"0x37e397fc7c91f5e4\",2],[\"0xd2bc9897eed08f15\",3],[\"0x40fe3ad401f8959a\",6],\ [\"0xbc9d89904f5b923f\",1],[\"0xc6e9a76309f39b09\",2],[\"0xdd718d5cc53262d4\",1],\ [\"0xcbca25e39f142387\",2],[\"0xf78b278be53f454c\",2],[\"0xab3c0572291feb8b\",1],\ diff --git a/substrate/client/transaction-pool/src/lib.rs b/substrate/client/transaction-pool/src/lib.rs index faa3f455a580..64b301e6bf36 100644 --- a/substrate/client/transaction-pool/src/lib.rs +++ b/substrate/client/transaction-pool/src/lib.rs @@ -164,8 +164,9 @@ where pool_api: Arc, best_block_hash: Block::Hash, finalized_hash: Block::Hash, + options: graph::Options, ) -> (Self, Pin + Send>>) { - let pool = Arc::new(graph::Pool::new(Default::default(), true.into(), pool_api.clone())); + let pool = Arc::new(graph::Pool::new(options, true.into(), pool_api.clone())); let (revalidation_queue, background_task) = revalidation::RevalidationQueue::new_background( pool_api.clone(), pool.clone(), diff --git a/substrate/client/transaction-pool/tests/pool.rs b/substrate/client/transaction-pool/tests/pool.rs index 6b1a197440c1..461b9860d414 100644 --- a/substrate/client/transaction-pool/tests/pool.rs +++ b/substrate/client/transaction-pool/tests/pool.rs @@ -73,7 +73,7 @@ fn create_basic_pool_with_genesis( .map(|blocks| blocks[0].0.header.hash()) .expect("there is block 0. qed") }; - BasicPool::new_test(test_api, genesis_hash, genesis_hash) + BasicPool::new_test(test_api, genesis_hash, genesis_hash, Default::default()) } fn create_basic_pool(test_api: TestApi) -> BasicPool { @@ -994,6 +994,7 @@ fn import_notification_to_pool_maintain_works() { )), best_hash, finalized_hash, + Default::default(), ) .0, ); diff --git a/substrate/frame/contracts/src/wasm/mod.rs b/substrate/frame/contracts/src/wasm/mod.rs index c96c28565095..34b45cba5d28 100644 --- a/substrate/frame/contracts/src/wasm/mod.rs +++ b/substrate/frame/contracts/src/wasm/mod.rs @@ -1821,17 +1821,6 @@ mod tests { assert!(weight_left.all_gt(actual_left), "gas_left must be greater than final"); } - /// Test that [`frame_support::weights::OldWeight`] en/decodes the same as our - /// [`crate::OldWeight`]. - #[test] - fn old_weight_decode() { - #![allow(deprecated)] - let sp = frame_support::weights::OldWeight(42).encode(); - let our = crate::OldWeight::decode(&mut &*sp).unwrap(); - - assert_eq!(our, 42); - } - const CODE_VALUE_TRANSFERRED: &str = r#" (module (import "seal0" "seal_value_transferred" (func $seal_value_transferred (param i32 i32))) diff --git a/substrate/frame/executive/Cargo.toml b/substrate/frame/executive/Cargo.toml index a4ca265f6178..63285e4cb493 100644 --- a/substrate/frame/executive/Cargo.toml +++ b/substrate/frame/executive/Cargo.toml @@ -16,6 +16,7 @@ workspace = true targets = ["x86_64-unknown-linux-gnu"] [dependencies] +aquamarine = "0.3.2" codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = [ "derive", ] } @@ -44,6 +45,7 @@ default = ["std"] with-tracing = ["sp-tracing/with-tracing"] std = [ "codec/std", + "frame-support/experimental", "frame-support/std", "frame-system/std", "frame-try-runtime/std", diff --git a/substrate/frame/executive/src/lib.rs b/substrate/frame/executive/src/lib.rs index 48ff675f8082..3028eaf318e0 100644 --- a/substrate/frame/executive/src/lib.rs +++ b/substrate/frame/executive/src/lib.rs @@ -15,6 +15,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![cfg_attr(not(feature = "std"), no_std)] + //! # Executive Module //! //! The Executive module acts as the orchestration layer for the runtime. It dispatches incoming @@ -35,6 +37,8 @@ //! - Finalize a block. //! - Start an off-chain worker. //! +//! The flow of their application in a block is explained in the [block flowchart](block_flowchart). +//! //! ### Implementations //! //! The Executive module provides the following implementations: @@ -114,17 +118,51 @@ //! pub type Executive = executive::Executive; //! ``` -#![cfg_attr(not(feature = "std"), no_std)] +#[cfg(doc)] +#[cfg_attr(doc, aquamarine::aquamarine)] +/// # Block Execution +/// +/// These are the steps of block execution as done by [`Executive::execute_block`]. A block is +/// invalid if any of them fail. +/// +/// ```mermaid +/// flowchart TD +/// Executive::execute_block --> on_runtime_upgrade +/// on_runtime_upgrade --> System::initialize +/// Executive::initialize_block --> System::initialize +/// System::initialize --> on_initialize +/// on_initialize --> PreInherents[System::PreInherents] +/// PreInherents --> Inherents[Apply Inherents] +/// Inherents --> PostInherents[System::PostInherents] +/// PostInherents --> Check{MBM ongoing?} +/// Check -->|No| poll +/// Check -->|Yes| post_transactions_2[System::PostTransaction] +/// post_transactions_2 --> Step[MBMs::step] +/// Step --> on_finalize +/// poll --> transactions[Apply Transactions] +/// transactions --> post_transactions_1[System::PostTransaction] +/// post_transactions_1 --> CheckIdle{Weight remaining?} +/// CheckIdle -->|Yes| on_idle +/// CheckIdle -->|No| on_finalize +/// on_idle --> on_finalize +/// ``` +pub mod block_flowchart {} + +#[cfg(test)] +mod tests; use codec::{Codec, Encode}; use frame_support::{ + defensive_assert, dispatch::{DispatchClass, DispatchInfo, GetDispatchInfo, PostDispatchInfo}, + migrations::MultiStepMigrator, pallet_prelude::InvalidTransaction, traits::{ BeforeAllRuntimeMigrations, EnsureInherentsAreFirst, ExecuteBlock, OffchainWorker, - OnFinalize, OnIdle, OnInitialize, OnRuntimeUpgrade, + OnFinalize, OnIdle, OnInitialize, OnPoll, OnRuntimeUpgrade, PostInherents, + PostTransactions, PreInherents, }, - weights::Weight, + weights::{Weight, WeightMeter}, }; use frame_system::pallet_prelude::BlockNumberFor; use sp_runtime::{ @@ -134,7 +172,7 @@ use sp_runtime::{ ValidateUnsigned, Zero, }, transaction_validity::{TransactionSource, TransactionValidity}, - ApplyExtrinsicResult, + ApplyExtrinsicResult, ExtrinsicInclusionMode, }; use sp_std::{marker::PhantomData, prelude::*}; @@ -198,7 +236,8 @@ impl< + OnInitialize> + OnIdle> + OnFinalize> - + OffchainWorker>, + + OffchainWorker> + + OnPoll>, COnRuntimeUpgrade: OnRuntimeUpgrade, > ExecuteBlock for Executive @@ -237,6 +276,7 @@ impl< + OnIdle> + OnFinalize> + OffchainWorker> + + OnPoll> + TryState> + TryDecodeEntireStorage, COnRuntimeUpgrade: OnRuntimeUpgrade, @@ -272,36 +312,50 @@ where select, ); - Self::initialize_block(block.header()); - Self::initial_checks(&block); - + let mode = Self::initialize_block(block.header()); + let num_inherents = Self::initial_checks(&block) as usize; let (header, extrinsics) = block.deconstruct(); + // Check if there are any forbidden non-inherents in the block. + if mode == ExtrinsicInclusionMode::OnlyInherents && extrinsics.len() > num_inherents { + return Err("Only inherents allowed".into()) + } + let try_apply_extrinsic = |uxt: Block::Extrinsic| -> ApplyExtrinsicResult { sp_io::init_tracing(); let encoded = uxt.encode(); let encoded_len = encoded.len(); + let is_inherent = System::is_inherent(&uxt); // skip signature verification. let xt = if signature_check { uxt.check(&Default::default()) } else { uxt.unchecked_into_checked_i_know_what_i_am_doing(&Default::default()) }?; - >::note_extrinsic(encoded); let dispatch_info = xt.get_dispatch_info(); + if !is_inherent && !>::inherents_applied() { + Self::inherents_applied(); + } + + >::note_extrinsic(encoded); let r = Applyable::apply::(xt, &dispatch_info, encoded_len)?; + if r.is_err() && dispatch_info.class == DispatchClass::Mandatory { + return Err(InvalidTransaction::BadMandatory.into()) + } + >::note_applied_extrinsic(&r, dispatch_info); Ok(r.map(|_| ()).map_err(|e| e.error)) }; - for e in extrinsics { + // Apply extrinsics: + for e in extrinsics.iter() { if let Err(err) = try_apply_extrinsic(e.clone()) { log::error!( - target: LOG_TARGET, "executing transaction {:?} failed due to {:?}. Aborting the rest of the block execution.", + target: LOG_TARGET, "transaction {:?} failed due to {:?}. Aborting the rest of the block execution.", e, err, ); @@ -309,9 +363,17 @@ where } } + // In this case there were no transactions to trigger this state transition: + if !>::inherents_applied() { + Self::inherents_applied(); + } + // post-extrinsics book-keeping >::note_finished_extrinsics(); - Self::idle_and_finalize_hook(*header.number()); + ::PostTransactions::post_transactions(); + + Self::on_idle_hook(*header.number()); + Self::on_finalize_hook(*header.number()); // run the try-state checks of all pallets, ensuring they don't alter any state. let _guard = frame_support::StorageNoopGuard::default(); @@ -449,7 +511,8 @@ impl< + OnInitialize> + OnIdle> + OnFinalize> - + OffchainWorker>, + + OffchainWorker> + + OnPoll>, COnRuntimeUpgrade: OnRuntimeUpgrade, > Executive where @@ -464,16 +527,36 @@ where pub fn execute_on_runtime_upgrade() -> Weight { let before_all_weight = ::before_all_runtime_migrations(); - <(COnRuntimeUpgrade, AllPalletsWithSystem) as OnRuntimeUpgrade>::on_runtime_upgrade() - .saturating_add(before_all_weight) + + let runtime_upgrade_weight = <( + COnRuntimeUpgrade, + ::SingleBlockMigrations, + // We want to run the migrations before we call into the pallets as they may + // access any state that would then not be migrated. + AllPalletsWithSystem, + ) as OnRuntimeUpgrade>::on_runtime_upgrade(); + + before_all_weight.saturating_add(runtime_upgrade_weight) } /// Start the execution of a particular block. - pub fn initialize_block(header: &frame_system::pallet_prelude::HeaderFor) { + pub fn initialize_block( + header: &frame_system::pallet_prelude::HeaderFor, + ) -> ExtrinsicInclusionMode { sp_io::init_tracing(); sp_tracing::enter_span!(sp_tracing::Level::TRACE, "init_block"); let digests = Self::extract_pre_digest(header); Self::initialize_block_impl(header.number(), header.parent_hash(), &digests); + + Self::extrinsic_mode() + } + + fn extrinsic_mode() -> ExtrinsicInclusionMode { + if ::MultiBlockMigrator::ongoing() { + ExtrinsicInclusionMode::OnlyInherents + } else { + ExtrinsicInclusionMode::AllExtrinsics + } } fn extract_pre_digest(header: &frame_system::pallet_prelude::HeaderFor) -> Digest { @@ -519,6 +602,7 @@ where ); frame_system::Pallet::::note_finished_initialize(); + ::PreInherents::pre_inherents(); } /// Returns if the runtime has been upgraded, based on [`frame_system::LastRuntimeUpgrade`]. @@ -529,7 +613,8 @@ where last.map(|v| v.was_upgraded(¤t)).unwrap_or(true) } - fn initial_checks(block: &Block) { + /// Returns the number of inherents in the block. + fn initial_checks(block: &Block) -> u32 { sp_tracing::enter_span!(sp_tracing::Level::TRACE, "initial_checks"); let header = block.header(); @@ -542,8 +627,9 @@ where "Parent hash should be valid.", ); - if let Err(i) = System::ensure_inherents_are_first(block) { - panic!("Invalid inherent position for extrinsic at index {}", i); + match System::ensure_inherents_are_first(block) { + Ok(num) => num, + Err(i) => panic!("Invalid inherent position for extrinsic at index {}", i), } } @@ -552,53 +638,90 @@ where sp_io::init_tracing(); sp_tracing::within_span! { sp_tracing::info_span!("execute_block", ?block); + // Execute `on_runtime_upgrade` and `on_initialize`. + let mode = Self::initialize_block(block.header()); + let num_inherents = Self::initial_checks(&block) as usize; + let (header, extrinsics) = block.deconstruct(); + let num_extrinsics = extrinsics.len(); - Self::initialize_block(block.header()); + if mode == ExtrinsicInclusionMode::OnlyInherents && num_extrinsics > num_inherents { + // Invalid block + panic!("Only inherents are allowed in this block") + } - // any initial checks - Self::initial_checks(&block); + Self::apply_extrinsics(extrinsics.into_iter()); - // execute extrinsics - let (header, extrinsics) = block.deconstruct(); - Self::execute_extrinsics_with_book_keeping(extrinsics, *header.number()); + // In this case there were no transactions to trigger this state transition: + if !>::inherents_applied() { + defensive_assert!(num_inherents == num_extrinsics); + Self::inherents_applied(); + } - // any final checks + >::note_finished_extrinsics(); + ::PostTransactions::post_transactions(); + + Self::on_idle_hook(*header.number()); + Self::on_finalize_hook(*header.number()); Self::final_checks(&header); } } - /// Execute given extrinsics and take care of post-extrinsics book-keeping. - fn execute_extrinsics_with_book_keeping( - extrinsics: Vec, - block_number: NumberFor, - ) { + /// Logic that runs directly after inherent application. + /// + /// It advances the Multi-Block-Migrations or runs the `on_poll` hook. + pub fn inherents_applied() { + >::note_inherents_applied(); + ::PostInherents::post_inherents(); + + if ::MultiBlockMigrator::ongoing() { + let used_weight = ::MultiBlockMigrator::step(); + >::register_extra_weight_unchecked( + used_weight, + DispatchClass::Mandatory, + ); + } else { + let block_number = >::block_number(); + Self::on_poll_hook(block_number); + } + } + + /// Execute given extrinsics. + fn apply_extrinsics(extrinsics: impl Iterator) { extrinsics.into_iter().for_each(|e| { if let Err(e) = Self::apply_extrinsic(e) { let err: &'static str = e.into(); panic!("{}", err) } }); - - // post-extrinsics book-keeping - >::note_finished_extrinsics(); - - Self::idle_and_finalize_hook(block_number); } /// Finalize the block - it is up the caller to ensure that all header fields are valid /// except state-root. + // Note: Only used by the block builder - not Executive itself. pub fn finalize_block() -> frame_system::pallet_prelude::HeaderFor { sp_io::init_tracing(); sp_tracing::enter_span!(sp_tracing::Level::TRACE, "finalize_block"); - >::note_finished_extrinsics(); - let block_number = >::block_number(); - Self::idle_and_finalize_hook(block_number); + // In this case there were no transactions to trigger this state transition: + if !>::inherents_applied() { + Self::inherents_applied(); + } + >::note_finished_extrinsics(); + ::PostTransactions::post_transactions(); + let block_number = >::block_number(); + Self::on_idle_hook(block_number); + Self::on_finalize_hook(block_number); >::finalize() } - fn idle_and_finalize_hook(block_number: NumberFor) { + /// Run the `on_idle` hook of all pallet, but only if there is weight remaining and there are no + /// ongoing MBMs. + fn on_idle_hook(block_number: NumberFor) { + if ::MultiBlockMigrator::ongoing() { + return + } + let weight = >::block_weight(); let max_weight = >::get().max_block; let remaining_weight = max_weight.saturating_sub(weight.total()); @@ -613,7 +736,33 @@ where DispatchClass::Mandatory, ); } + } + + fn on_poll_hook(block_number: NumberFor) { + defensive_assert!( + !::MultiBlockMigrator::ongoing(), + "on_poll should not be called during migrations" + ); + let weight = >::block_weight(); + let max_weight = >::get().max_block; + let remaining = max_weight.saturating_sub(weight.total()); + + if remaining.all_gt(Weight::zero()) { + let mut meter = WeightMeter::with_limit(remaining); + >>::on_poll( + block_number, + &mut meter, + ); + >::register_extra_weight_unchecked( + meter.consumed(), + DispatchClass::Mandatory, + ); + } + } + + /// Run the `on_finalize` hook of all pallet. + fn on_finalize_hook(block_number: NumberFor) { >>::on_finalize(block_number); } @@ -627,8 +776,18 @@ where let encoded_len = encoded.len(); sp_tracing::enter_span!(sp_tracing::info_span!("apply_extrinsic", ext=?sp_core::hexdisplay::HexDisplay::from(&encoded))); + + // We use the dedicated `is_inherent` check here, since just relying on `Mandatory` dispatch + // class does not capture optional inherents. + let is_inherent = System::is_inherent(&uxt); + // Verify that the signature is good. let xt = uxt.check(&Default::default())?; + let dispatch_info = xt.get_dispatch_info(); + + if !is_inherent && !>::inherents_applied() { + Self::inherents_applied(); + } // We don't need to make sure to `note_extrinsic` only after we know it's going to be // executed to prevent it from leaking in storage since at this point, it will either @@ -637,8 +796,6 @@ where // AUDIT: Under no circumstances may this function panic from here onwards. - // Decode parameters and dispatch - let dispatch_info = xt.get_dispatch_info(); let r = Applyable::apply::(xt, &dispatch_info, encoded_len)?; // Mandatory(inherents) are not allowed to fail. @@ -745,956 +902,3 @@ where ) } } - -#[cfg(test)] -mod tests { - use super::*; - - use sp_core::H256; - use sp_runtime::{ - generic::{DigestItem, Era}, - testing::{Block, Digest, Header}, - traits::{BlakeTwo256, Block as BlockT, Header as HeaderT, IdentityLookup}, - transaction_validity::{ - InvalidTransaction, TransactionValidityError, UnknownTransaction, ValidTransaction, - }, - BuildStorage, DispatchError, - }; - - use frame_support::{ - assert_err, derive_impl, parameter_types, - traits::{fungible, ConstU32, ConstU64, ConstU8, Currency}, - weights::{ConstantMultiplier, IdentityFee, RuntimeDbWeight, Weight, WeightToFee}, - }; - use frame_system::{ChainContext, LastRuntimeUpgrade, LastRuntimeUpgradeInfo}; - use pallet_balances::Call as BalancesCall; - use pallet_transaction_payment::CurrencyAdapter; - - const TEST_KEY: &[u8] = b":test:key:"; - - #[frame_support::pallet(dev_mode)] - mod custom { - use frame_support::pallet_prelude::*; - use frame_system::pallet_prelude::*; - - #[pallet::pallet] - pub struct Pallet(_); - - #[pallet::config] - pub trait Config: frame_system::Config {} - - #[pallet::hooks] - impl Hooks> for Pallet { - // module hooks. - // one with block number arg and one without - fn on_initialize(n: BlockNumberFor) -> Weight { - println!("on_initialize({})", n); - Weight::from_parts(175, 0) - } - - fn on_idle(n: BlockNumberFor, remaining_weight: Weight) -> Weight { - println!("on_idle{}, {})", n, remaining_weight); - Weight::from_parts(175, 0) - } - - fn on_finalize(n: BlockNumberFor) { - println!("on_finalize({})", n); - } - - fn on_runtime_upgrade() -> Weight { - sp_io::storage::set(super::TEST_KEY, "module".as_bytes()); - Weight::from_parts(200, 0) - } - - fn offchain_worker(n: BlockNumberFor) { - assert_eq!(BlockNumberFor::::from(1u32), n); - } - } - - #[pallet::call] - impl Pallet { - pub fn some_function(origin: OriginFor) -> DispatchResult { - // NOTE: does not make any different. - frame_system::ensure_signed(origin)?; - Ok(()) - } - - #[pallet::weight((200, DispatchClass::Operational))] - pub fn some_root_operation(origin: OriginFor) -> DispatchResult { - frame_system::ensure_root(origin)?; - Ok(()) - } - - pub fn some_unsigned_message(origin: OriginFor) -> DispatchResult { - frame_system::ensure_none(origin)?; - Ok(()) - } - - pub fn allowed_unsigned(origin: OriginFor) -> DispatchResult { - frame_system::ensure_root(origin)?; - Ok(()) - } - - pub fn unallowed_unsigned(origin: OriginFor) -> DispatchResult { - frame_system::ensure_root(origin)?; - Ok(()) - } - - #[pallet::weight((0, DispatchClass::Mandatory))] - pub fn inherent_call(origin: OriginFor) -> DispatchResult { - frame_system::ensure_none(origin)?; - Ok(()) - } - - pub fn calculate_storage_root(_origin: OriginFor) -> DispatchResult { - let root = sp_io::storage::root(sp_runtime::StateVersion::V1); - sp_io::storage::set("storage_root".as_bytes(), &root); - Ok(()) - } - } - - #[pallet::inherent] - impl ProvideInherent for Pallet { - type Call = Call; - - type Error = sp_inherents::MakeFatalError<()>; - - const INHERENT_IDENTIFIER: [u8; 8] = *b"test1234"; - - fn create_inherent(_data: &InherentData) -> Option { - None - } - - fn is_inherent(call: &Self::Call) -> bool { - *call == Call::::inherent_call {} - } - } - - #[pallet::validate_unsigned] - impl ValidateUnsigned for Pallet { - type Call = Call; - - // Inherent call is accepted for being dispatched - fn pre_dispatch(call: &Self::Call) -> Result<(), TransactionValidityError> { - match call { - Call::allowed_unsigned { .. } => Ok(()), - Call::inherent_call { .. } => Ok(()), - _ => Err(UnknownTransaction::NoUnsignedValidator.into()), - } - } - - // Inherent call is not validated as unsigned - fn validate_unsigned( - _source: TransactionSource, - call: &Self::Call, - ) -> TransactionValidity { - match call { - Call::allowed_unsigned { .. } => Ok(Default::default()), - _ => UnknownTransaction::NoUnsignedValidator.into(), - } - } - } - } - - frame_support::construct_runtime!( - pub enum Runtime { - System: frame_system, - Balances: pallet_balances, - TransactionPayment: pallet_transaction_payment, - Custom: custom, - } - ); - - parameter_types! { - pub BlockWeights: frame_system::limits::BlockWeights = - frame_system::limits::BlockWeights::builder() - .base_block(Weight::from_parts(10, 0)) - .for_class(DispatchClass::all(), |weights| weights.base_extrinsic = Weight::from_parts(5, 0)) - .for_class(DispatchClass::non_mandatory(), |weights| weights.max_total = Weight::from_parts(1024, u64::MAX).into()) - .build_or_panic(); - pub const DbWeight: RuntimeDbWeight = RuntimeDbWeight { - read: 10, - write: 100, - }; - } - #[derive_impl(frame_system::config_preludes::TestDefaultConfig as frame_system::DefaultConfig)] - impl frame_system::Config for Runtime { - type BaseCallFilter = frame_support::traits::Everything; - type BlockWeights = BlockWeights; - type BlockLength = (); - type DbWeight = (); - type RuntimeOrigin = RuntimeOrigin; - type Nonce = u64; - type RuntimeCall = RuntimeCall; - type Hash = sp_core::H256; - type Hashing = BlakeTwo256; - type AccountId = u64; - type Lookup = IdentityLookup; - type Block = TestBlock; - type RuntimeEvent = RuntimeEvent; - type BlockHashCount = ConstU64<250>; - type Version = RuntimeVersion; - type PalletInfo = PalletInfo; - type AccountData = pallet_balances::AccountData; - type OnNewAccount = (); - type OnKilledAccount = (); - type SystemWeightInfo = (); - type SS58Prefix = (); - type OnSetCode = (); - type MaxConsumers = ConstU32<16>; - } - - type Balance = u64; - impl pallet_balances::Config for Runtime { - type Balance = Balance; - type RuntimeEvent = RuntimeEvent; - type DustRemoval = (); - type ExistentialDeposit = ConstU64<1>; - type AccountStore = System; - type MaxLocks = (); - type MaxReserves = (); - type ReserveIdentifier = [u8; 8]; - type WeightInfo = (); - type FreezeIdentifier = (); - type MaxFreezes = ConstU32<1>; - type RuntimeHoldReason = (); - type RuntimeFreezeReason = (); - } - - parameter_types! { - pub const TransactionByteFee: Balance = 0; - } - impl pallet_transaction_payment::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type OnChargeTransaction = CurrencyAdapter; - type OperationalFeeMultiplier = ConstU8<5>; - type WeightToFee = IdentityFee; - type LengthToFee = ConstantMultiplier; - type FeeMultiplierUpdate = (); - } - impl custom::Config for Runtime {} - - pub struct RuntimeVersion; - impl frame_support::traits::Get for RuntimeVersion { - fn get() -> sp_version::RuntimeVersion { - RuntimeVersionTestValues::get().clone() - } - } - - parameter_types! { - pub static RuntimeVersionTestValues: sp_version::RuntimeVersion = - Default::default(); - } - - type SignedExtra = ( - frame_system::CheckEra, - frame_system::CheckNonce, - frame_system::CheckWeight, - pallet_transaction_payment::ChargeTransactionPayment, - ); - type TestXt = sp_runtime::testing::TestXt; - type TestBlock = Block; - - // Will contain `true` when the custom runtime logic was called. - const CUSTOM_ON_RUNTIME_KEY: &[u8] = b":custom:on_runtime"; - - struct CustomOnRuntimeUpgrade; - impl OnRuntimeUpgrade for CustomOnRuntimeUpgrade { - fn on_runtime_upgrade() -> Weight { - sp_io::storage::set(TEST_KEY, "custom_upgrade".as_bytes()); - sp_io::storage::set(CUSTOM_ON_RUNTIME_KEY, &true.encode()); - System::deposit_event(frame_system::Event::CodeUpdated); - - assert_eq!(0, System::last_runtime_upgrade_spec_version()); - - Weight::from_parts(100, 0) - } - } - - type Executive = super::Executive< - Runtime, - Block, - ChainContext, - Runtime, - AllPalletsWithSystem, - CustomOnRuntimeUpgrade, - >; - - fn extra(nonce: u64, fee: Balance) -> SignedExtra { - ( - frame_system::CheckEra::from(Era::Immortal), - frame_system::CheckNonce::from(nonce), - frame_system::CheckWeight::new(), - pallet_transaction_payment::ChargeTransactionPayment::from(fee), - ) - } - - fn sign_extra(who: u64, nonce: u64, fee: Balance) -> Option<(u64, SignedExtra)> { - Some((who, extra(nonce, fee))) - } - - fn call_transfer(dest: u64, value: u64) -> RuntimeCall { - RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest, value }) - } - - #[test] - fn balance_transfer_dispatch_works() { - let mut t = frame_system::GenesisConfig::::default().build_storage().unwrap(); - pallet_balances::GenesisConfig:: { balances: vec![(1, 211)] } - .assimilate_storage(&mut t) - .unwrap(); - let xt = TestXt::new(call_transfer(2, 69), sign_extra(1, 0, 0)); - let weight = xt.get_dispatch_info().weight + - ::BlockWeights::get() - .get(DispatchClass::Normal) - .base_extrinsic; - let fee: Balance = - ::WeightToFee::weight_to_fee(&weight); - let mut t = sp_io::TestExternalities::new(t); - t.execute_with(|| { - Executive::initialize_block(&Header::new( - 1, - H256::default(), - H256::default(), - [69u8; 32].into(), - Digest::default(), - )); - let r = Executive::apply_extrinsic(xt); - assert!(r.is_ok()); - assert_eq!(>::total_balance(&1), 142 - fee); - assert_eq!(>::total_balance(&2), 69); - }); - } - - fn new_test_ext(balance_factor: Balance) -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::::default().build_storage().unwrap(); - pallet_balances::GenesisConfig:: { balances: vec![(1, 111 * balance_factor)] } - .assimilate_storage(&mut t) - .unwrap(); - t.into() - } - - fn new_test_ext_v0(balance_factor: Balance) -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::::default().build_storage().unwrap(); - pallet_balances::GenesisConfig:: { balances: vec![(1, 111 * balance_factor)] } - .assimilate_storage(&mut t) - .unwrap(); - (t, sp_runtime::StateVersion::V0).into() - } - - #[test] - fn block_import_works() { - block_import_works_inner( - new_test_ext_v0(1), - array_bytes::hex_n_into_unchecked( - "65e953676859e7a33245908af7ad3637d6861eb90416d433d485e95e2dd174a1", - ), - ); - block_import_works_inner( - new_test_ext(1), - array_bytes::hex_n_into_unchecked( - "5a19b3d6fdb7241836349fdcbe2d9df4d4f945b949d979e31ad50bff1cbcd1c2", - ), - ); - } - fn block_import_works_inner(mut ext: sp_io::TestExternalities, state_root: H256) { - ext.execute_with(|| { - Executive::execute_block(Block { - header: Header { - parent_hash: [69u8; 32].into(), - number: 1, - state_root, - extrinsics_root: array_bytes::hex_n_into_unchecked( - "03170a2e7597b7b7e3d84c05391d139a62b157e78786d8c082f29dcf4c111314", - ), - digest: Digest { logs: vec![] }, - }, - extrinsics: vec![], - }); - }); - } - - #[test] - #[should_panic] - fn block_import_of_bad_state_root_fails() { - new_test_ext(1).execute_with(|| { - Executive::execute_block(Block { - header: Header { - parent_hash: [69u8; 32].into(), - number: 1, - state_root: [0u8; 32].into(), - extrinsics_root: array_bytes::hex_n_into_unchecked( - "03170a2e7597b7b7e3d84c05391d139a62b157e78786d8c082f29dcf4c111314", - ), - digest: Digest { logs: vec![] }, - }, - extrinsics: vec![], - }); - }); - } - - #[test] - #[should_panic] - fn block_import_of_bad_extrinsic_root_fails() { - new_test_ext(1).execute_with(|| { - Executive::execute_block(Block { - header: Header { - parent_hash: [69u8; 32].into(), - number: 1, - state_root: array_bytes::hex_n_into_unchecked( - "75e7d8f360d375bbe91bcf8019c01ab6362448b4a89e3b329717eb9d910340e5", - ), - extrinsics_root: [0u8; 32].into(), - digest: Digest { logs: vec![] }, - }, - extrinsics: vec![], - }); - }); - } - - #[test] - fn bad_extrinsic_not_inserted() { - let mut t = new_test_ext(1); - // bad nonce check! - let xt = TestXt::new(call_transfer(33, 69), sign_extra(1, 30, 0)); - t.execute_with(|| { - Executive::initialize_block(&Header::new( - 1, - H256::default(), - H256::default(), - [69u8; 32].into(), - Digest::default(), - )); - assert_err!( - Executive::apply_extrinsic(xt), - TransactionValidityError::Invalid(InvalidTransaction::Future) - ); - assert_eq!(>::extrinsic_index(), Some(0)); - }); - } - - #[test] - fn block_weight_limit_enforced() { - let mut t = new_test_ext(10000); - // given: TestXt uses the encoded len as fixed Len: - let xt = TestXt::new( - RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest: 33, value: 0 }), - sign_extra(1, 0, 0), - ); - let encoded = xt.encode(); - let encoded_len = encoded.len() as u64; - // on_initialize weight + base block execution weight - let block_weights = ::BlockWeights::get(); - let base_block_weight = Weight::from_parts(175, 0) + block_weights.base_block; - let limit = block_weights.get(DispatchClass::Normal).max_total.unwrap() - base_block_weight; - let num_to_exhaust_block = limit.ref_time() / (encoded_len + 5); - t.execute_with(|| { - Executive::initialize_block(&Header::new( - 1, - H256::default(), - H256::default(), - [69u8; 32].into(), - Digest::default(), - )); - // Base block execution weight + `on_initialize` weight from the custom module. - assert_eq!(>::block_weight().total(), base_block_weight); - - for nonce in 0..=num_to_exhaust_block { - let xt = TestXt::new( - RuntimeCall::Balances(BalancesCall::transfer_allow_death { - dest: 33, - value: 0, - }), - sign_extra(1, nonce.into(), 0), - ); - let res = Executive::apply_extrinsic(xt); - if nonce != num_to_exhaust_block { - assert!(res.is_ok()); - assert_eq!( - >::block_weight().total(), - //--------------------- on_initialize + block_execution + extrinsic_base weight - Weight::from_parts((encoded_len + 5) * (nonce + 1), 0) + base_block_weight, - ); - assert_eq!( - >::extrinsic_index(), - Some(nonce as u32 + 1) - ); - } else { - assert_eq!(res, Err(InvalidTransaction::ExhaustsResources.into())); - } - } - }); - } - - #[test] - fn block_weight_and_size_is_stored_per_tx() { - let xt = TestXt::new( - RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest: 33, value: 0 }), - sign_extra(1, 0, 0), - ); - let x1 = TestXt::new( - RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest: 33, value: 0 }), - sign_extra(1, 1, 0), - ); - let x2 = TestXt::new( - RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest: 33, value: 0 }), - sign_extra(1, 2, 0), - ); - let len = xt.clone().encode().len() as u32; - let mut t = new_test_ext(1); - t.execute_with(|| { - // Block execution weight + on_initialize weight from custom module - let base_block_weight = Weight::from_parts(175, 0) + - ::BlockWeights::get().base_block; - - Executive::initialize_block(&Header::new( - 1, - H256::default(), - H256::default(), - [69u8; 32].into(), - Digest::default(), - )); - - assert_eq!(>::block_weight().total(), base_block_weight); - assert_eq!(>::all_extrinsics_len(), 0); - - assert!(Executive::apply_extrinsic(xt.clone()).unwrap().is_ok()); - assert!(Executive::apply_extrinsic(x1.clone()).unwrap().is_ok()); - assert!(Executive::apply_extrinsic(x2.clone()).unwrap().is_ok()); - - // default weight for `TestXt` == encoded length. - let extrinsic_weight = Weight::from_parts(len as u64, 0) + - ::BlockWeights::get() - .get(DispatchClass::Normal) - .base_extrinsic; - assert_eq!( - >::block_weight().total(), - base_block_weight + 3u64 * extrinsic_weight, - ); - assert_eq!(>::all_extrinsics_len(), 3 * len); - - let _ = >::finalize(); - // All extrinsics length cleaned on `System::finalize` - assert_eq!(>::all_extrinsics_len(), 0); - - // New Block - Executive::initialize_block(&Header::new( - 2, - H256::default(), - H256::default(), - [69u8; 32].into(), - Digest::default(), - )); - - // Block weight cleaned up on `System::initialize` - assert_eq!(>::block_weight().total(), base_block_weight); - }); - } - - #[test] - fn validate_unsigned() { - let valid = TestXt::new(RuntimeCall::Custom(custom::Call::allowed_unsigned {}), None); - let invalid = TestXt::new(RuntimeCall::Custom(custom::Call::unallowed_unsigned {}), None); - let mut t = new_test_ext(1); - - t.execute_with(|| { - assert_eq!( - Executive::validate_transaction( - TransactionSource::InBlock, - valid.clone(), - Default::default(), - ), - Ok(ValidTransaction::default()), - ); - assert_eq!( - Executive::validate_transaction( - TransactionSource::InBlock, - invalid.clone(), - Default::default(), - ), - Err(TransactionValidityError::Unknown(UnknownTransaction::NoUnsignedValidator)), - ); - assert_eq!(Executive::apply_extrinsic(valid), Ok(Err(DispatchError::BadOrigin))); - assert_eq!( - Executive::apply_extrinsic(invalid), - Err(TransactionValidityError::Unknown(UnknownTransaction::NoUnsignedValidator)) - ); - }); - } - - #[test] - fn can_not_pay_for_tx_fee_on_full_lock() { - let mut t = new_test_ext(1); - t.execute_with(|| { - as fungible::MutateFreeze>::set_freeze( - &(), - &1, - 110, - ) - .unwrap(); - let xt = TestXt::new( - RuntimeCall::System(frame_system::Call::remark { remark: vec![1u8] }), - sign_extra(1, 0, 0), - ); - Executive::initialize_block(&Header::new( - 1, - H256::default(), - H256::default(), - [69u8; 32].into(), - Digest::default(), - )); - - assert_eq!(Executive::apply_extrinsic(xt), Err(InvalidTransaction::Payment.into()),); - assert_eq!(>::total_balance(&1), 111); - }); - } - - #[test] - fn block_hooks_weight_is_stored() { - new_test_ext(1).execute_with(|| { - Executive::initialize_block(&Header::new_from_number(1)); - Executive::finalize_block(); - // NOTE: might need updates over time if new weights are introduced. - // For now it only accounts for the base block execution weight and - // the `on_initialize` weight defined in the custom test module. - assert_eq!( - >::block_weight().total(), - Weight::from_parts(175 + 175 + 10, 0) - ); - }) - } - - #[test] - fn runtime_upgraded_should_work() { - new_test_ext(1).execute_with(|| { - RuntimeVersionTestValues::mutate(|v| *v = Default::default()); - // It should be added at genesis - assert!(LastRuntimeUpgrade::::exists()); - assert!(!Executive::runtime_upgraded()); - - RuntimeVersionTestValues::mutate(|v| { - *v = sp_version::RuntimeVersion { spec_version: 1, ..Default::default() } - }); - assert!(Executive::runtime_upgraded()); - - RuntimeVersionTestValues::mutate(|v| { - *v = sp_version::RuntimeVersion { - spec_version: 1, - spec_name: "test".into(), - ..Default::default() - } - }); - assert!(Executive::runtime_upgraded()); - - RuntimeVersionTestValues::mutate(|v| { - *v = sp_version::RuntimeVersion { - spec_version: 0, - impl_version: 2, - ..Default::default() - } - }); - assert!(!Executive::runtime_upgraded()); - - LastRuntimeUpgrade::::take(); - assert!(Executive::runtime_upgraded()); - }) - } - - #[test] - fn last_runtime_upgrade_was_upgraded_works() { - let test_data = vec![ - (0, "", 1, "", true), - (1, "", 1, "", false), - (1, "", 1, "test", true), - (1, "", 0, "", false), - (1, "", 0, "test", true), - ]; - - for (spec_version, spec_name, c_spec_version, c_spec_name, result) in test_data { - let current = sp_version::RuntimeVersion { - spec_version: c_spec_version, - spec_name: c_spec_name.into(), - ..Default::default() - }; - - let last = LastRuntimeUpgradeInfo { - spec_version: spec_version.into(), - spec_name: spec_name.into(), - }; - - assert_eq!(result, last.was_upgraded(¤t)); - } - } - - #[test] - fn custom_runtime_upgrade_is_called_before_modules() { - new_test_ext(1).execute_with(|| { - // Make sure `on_runtime_upgrade` is called. - RuntimeVersionTestValues::mutate(|v| { - *v = sp_version::RuntimeVersion { spec_version: 1, ..Default::default() } - }); - - Executive::initialize_block(&Header::new( - 1, - H256::default(), - H256::default(), - [69u8; 32].into(), - Digest::default(), - )); - - assert_eq!(&sp_io::storage::get(TEST_KEY).unwrap()[..], *b"module"); - assert_eq!(sp_io::storage::get(CUSTOM_ON_RUNTIME_KEY).unwrap(), true.encode()); - assert_eq!( - Some(RuntimeVersionTestValues::get().into()), - LastRuntimeUpgrade::::get(), - ) - }); - } - - #[test] - fn event_from_runtime_upgrade_is_included() { - new_test_ext(1).execute_with(|| { - // Make sure `on_runtime_upgrade` is called. - RuntimeVersionTestValues::mutate(|v| { - *v = sp_version::RuntimeVersion { spec_version: 1, ..Default::default() } - }); - - // set block number to non zero so events are not excluded - System::set_block_number(1); - - Executive::initialize_block(&Header::new( - 2, - H256::default(), - H256::default(), - [69u8; 32].into(), - Digest::default(), - )); - - System::assert_last_event(frame_system::Event::::CodeUpdated.into()); - }); - } - - /// Regression test that ensures that the custom on runtime upgrade is called when executive is - /// used through the `ExecuteBlock` trait. - #[test] - fn custom_runtime_upgrade_is_called_when_using_execute_block_trait() { - let xt = TestXt::new( - RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest: 33, value: 0 }), - sign_extra(1, 0, 0), - ); - - let header = new_test_ext(1).execute_with(|| { - // Make sure `on_runtime_upgrade` is called. - RuntimeVersionTestValues::mutate(|v| { - *v = sp_version::RuntimeVersion { spec_version: 1, ..Default::default() } - }); - - // Let's build some fake block. - Executive::initialize_block(&Header::new( - 1, - H256::default(), - H256::default(), - [69u8; 32].into(), - Digest::default(), - )); - - Executive::apply_extrinsic(xt.clone()).unwrap().unwrap(); - - Executive::finalize_block() - }); - - // Reset to get the correct new genesis below. - RuntimeVersionTestValues::mutate(|v| { - *v = sp_version::RuntimeVersion { spec_version: 0, ..Default::default() } - }); - - new_test_ext(1).execute_with(|| { - // Make sure `on_runtime_upgrade` is called. - RuntimeVersionTestValues::mutate(|v| { - *v = sp_version::RuntimeVersion { spec_version: 1, ..Default::default() } - }); - - >>::execute_block(Block::new(header, vec![xt])); - - assert_eq!(&sp_io::storage::get(TEST_KEY).unwrap()[..], *b"module"); - assert_eq!(sp_io::storage::get(CUSTOM_ON_RUNTIME_KEY).unwrap(), true.encode()); - }); - } - - #[test] - fn all_weights_are_recorded_correctly() { - // Reset to get the correct new genesis below. - RuntimeVersionTestValues::take(); - - new_test_ext(1).execute_with(|| { - // Make sure `on_runtime_upgrade` is called for maximum complexity - RuntimeVersionTestValues::mutate(|v| { - *v = sp_version::RuntimeVersion { spec_version: 1, ..Default::default() } - }); - - let block_number = 1; - - Executive::initialize_block(&Header::new( - block_number, - H256::default(), - H256::default(), - [69u8; 32].into(), - Digest::default(), - )); - - // Reset the last runtime upgrade info, to make the second call to `on_runtime_upgrade` - // succeed. - LastRuntimeUpgrade::::take(); - - // All weights that show up in the `initialize_block_impl` - let custom_runtime_upgrade_weight = CustomOnRuntimeUpgrade::on_runtime_upgrade(); - let runtime_upgrade_weight = - ::on_runtime_upgrade(); - let on_initialize_weight = - >::on_initialize(block_number); - let base_block_weight = - ::BlockWeights::get().base_block; - - // Weights are recorded correctly - assert_eq!( - frame_system::Pallet::::block_weight().total(), - custom_runtime_upgrade_weight + - runtime_upgrade_weight + - on_initialize_weight + base_block_weight, - ); - }); - } - - #[test] - fn offchain_worker_works_as_expected() { - new_test_ext(1).execute_with(|| { - let parent_hash = sp_core::H256::from([69u8; 32]); - let mut digest = Digest::default(); - digest.push(DigestItem::Seal([1, 2, 3, 4], vec![5, 6, 7, 8])); - - let header = - Header::new(1, H256::default(), H256::default(), parent_hash, digest.clone()); - - Executive::offchain_worker(&header); - - assert_eq!(digest, System::digest()); - assert_eq!(parent_hash, System::block_hash(0)); - assert_eq!(header.hash(), System::block_hash(1)); - }); - } - - #[test] - fn calculating_storage_root_twice_works() { - let call = RuntimeCall::Custom(custom::Call::calculate_storage_root {}); - let xt = TestXt::new(call, sign_extra(1, 0, 0)); - - let header = new_test_ext(1).execute_with(|| { - // Let's build some fake block. - Executive::initialize_block(&Header::new( - 1, - H256::default(), - H256::default(), - [69u8; 32].into(), - Digest::default(), - )); - - Executive::apply_extrinsic(xt.clone()).unwrap().unwrap(); - - Executive::finalize_block() - }); - - new_test_ext(1).execute_with(|| { - Executive::execute_block(Block::new(header, vec![xt])); - }); - } - - #[test] - #[should_panic(expected = "Invalid inherent position for extrinsic at index 1")] - fn invalid_inherent_position_fail() { - let xt1 = TestXt::new( - RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest: 33, value: 0 }), - sign_extra(1, 0, 0), - ); - let xt2 = TestXt::new(RuntimeCall::Custom(custom::Call::inherent_call {}), None); - - let header = new_test_ext(1).execute_with(|| { - // Let's build some fake block. - Executive::initialize_block(&Header::new( - 1, - H256::default(), - H256::default(), - [69u8; 32].into(), - Digest::default(), - )); - - Executive::apply_extrinsic(xt1.clone()).unwrap().unwrap(); - Executive::apply_extrinsic(xt2.clone()).unwrap().unwrap(); - - Executive::finalize_block() - }); - - new_test_ext(1).execute_with(|| { - Executive::execute_block(Block::new(header, vec![xt1, xt2])); - }); - } - - #[test] - fn valid_inherents_position_works() { - let xt1 = TestXt::new(RuntimeCall::Custom(custom::Call::inherent_call {}), None); - let xt2 = TestXt::new(call_transfer(33, 0), sign_extra(1, 0, 0)); - - let header = new_test_ext(1).execute_with(|| { - // Let's build some fake block. - Executive::initialize_block(&Header::new( - 1, - H256::default(), - H256::default(), - [69u8; 32].into(), - Digest::default(), - )); - - Executive::apply_extrinsic(xt1.clone()).unwrap().unwrap(); - Executive::apply_extrinsic(xt2.clone()).unwrap().unwrap(); - - Executive::finalize_block() - }); - - new_test_ext(1).execute_with(|| { - Executive::execute_block(Block::new(header, vec![xt1, xt2])); - }); - } - - #[test] - #[should_panic(expected = "A call was labelled as mandatory, but resulted in an Error.")] - fn invalid_inherents_fail_block_execution() { - let xt1 = - TestXt::new(RuntimeCall::Custom(custom::Call::inherent_call {}), sign_extra(1, 0, 0)); - - new_test_ext(1).execute_with(|| { - Executive::execute_block(Block::new( - Header::new( - 1, - H256::default(), - H256::default(), - [69u8; 32].into(), - Digest::default(), - ), - vec![xt1], - )); - }); - } - - // Inherents are created by the runtime and don't need to be validated. - #[test] - fn inherents_fail_validate_block() { - let xt1 = TestXt::new(RuntimeCall::Custom(custom::Call::inherent_call {}), None); - - new_test_ext(1).execute_with(|| { - assert_eq!( - Executive::validate_transaction(TransactionSource::External, xt1, H256::random()) - .unwrap_err(), - InvalidTransaction::MandatoryValidation.into() - ); - }) - } -} diff --git a/substrate/frame/executive/src/tests.rs b/substrate/frame/executive/src/tests.rs new file mode 100644 index 000000000000..70b55f6e8553 --- /dev/null +++ b/substrate/frame/executive/src/tests.rs @@ -0,0 +1,1389 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Test the `frame-executive` crate. + +use super::*; + +use sp_core::H256; +use sp_runtime::{ + generic::{DigestItem, Era}, + testing::{Block, Digest, Header}, + traits::{Block as BlockT, Header as HeaderT}, + transaction_validity::{ + InvalidTransaction, TransactionValidityError, UnknownTransaction, ValidTransaction, + }, + BuildStorage, DispatchError, +}; + +use frame_support::{ + assert_err, assert_ok, derive_impl, + migrations::MultiStepMigrator, + pallet_prelude::*, + parameter_types, + traits::{fungible, ConstU8, Currency, IsInherent}, + weights::{ConstantMultiplier, IdentityFee, RuntimeDbWeight, Weight, WeightMeter, WeightToFee}, +}; +use frame_system::{pallet_prelude::*, ChainContext, LastRuntimeUpgrade, LastRuntimeUpgradeInfo}; +use pallet_balances::Call as BalancesCall; +use pallet_transaction_payment::CurrencyAdapter; + +const TEST_KEY: &[u8] = b":test:key:"; + +#[frame_support::pallet(dev_mode)] +mod custom { + use super::*; + + #[pallet::pallet] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: frame_system::Config {} + + #[pallet::hooks] + impl Hooks> for Pallet { + // module hooks. + // one with block number arg and one without + fn on_initialize(_: BlockNumberFor) -> Weight { + Weight::from_parts(175, 0) + } + + fn on_idle(_: BlockNumberFor, _: Weight) -> Weight { + Weight::from_parts(175, 0) + } + + fn on_poll(_: BlockNumberFor, _: &mut WeightMeter) {} + + fn on_finalize(_: BlockNumberFor) {} + + fn on_runtime_upgrade() -> Weight { + sp_io::storage::set(super::TEST_KEY, "module".as_bytes()); + Weight::from_parts(200, 0) + } + + fn offchain_worker(n: BlockNumberFor) { + assert_eq!(BlockNumberFor::::from(1u32), n); + } + } + + #[pallet::call] + impl Pallet { + pub fn some_function(origin: OriginFor) -> DispatchResult { + // NOTE: does not make any difference. + frame_system::ensure_signed(origin)?; + Ok(()) + } + + #[pallet::weight((200, DispatchClass::Operational))] + pub fn some_root_operation(origin: OriginFor) -> DispatchResult { + frame_system::ensure_root(origin)?; + Ok(()) + } + + pub fn some_unsigned_message(origin: OriginFor) -> DispatchResult { + frame_system::ensure_none(origin)?; + Ok(()) + } + + pub fn allowed_unsigned(origin: OriginFor) -> DispatchResult { + frame_system::ensure_root(origin)?; + Ok(()) + } + + pub fn unallowed_unsigned(origin: OriginFor) -> DispatchResult { + frame_system::ensure_root(origin)?; + Ok(()) + } + + #[pallet::weight((0, DispatchClass::Mandatory))] + pub fn inherent(origin: OriginFor) -> DispatchResult { + frame_system::ensure_none(origin)?; + Ok(()) + } + + pub fn calculate_storage_root(_origin: OriginFor) -> DispatchResult { + let root = sp_io::storage::root(sp_runtime::StateVersion::V1); + sp_io::storage::set("storage_root".as_bytes(), &root); + Ok(()) + } + } + + #[pallet::inherent] + impl ProvideInherent for Pallet { + type Call = Call; + + type Error = sp_inherents::MakeFatalError<()>; + + const INHERENT_IDENTIFIER: [u8; 8] = *b"test1234"; + + fn create_inherent(_data: &InherentData) -> Option { + None + } + + fn is_inherent(call: &Self::Call) -> bool { + *call == Call::::inherent {} + } + } + + #[pallet::validate_unsigned] + impl ValidateUnsigned for Pallet { + type Call = Call; + + // Inherent call is accepted for being dispatched + fn pre_dispatch(call: &Self::Call) -> Result<(), TransactionValidityError> { + match call { + Call::allowed_unsigned { .. } => Ok(()), + Call::inherent { .. } => Ok(()), + _ => Err(UnknownTransaction::NoUnsignedValidator.into()), + } + } + + // Inherent call is not validated as unsigned + fn validate_unsigned(_source: TransactionSource, call: &Self::Call) -> TransactionValidity { + match call { + Call::allowed_unsigned { .. } => Ok(Default::default()), + _ => UnknownTransaction::NoUnsignedValidator.into(), + } + } + } +} + +#[frame_support::pallet(dev_mode)] +mod custom2 { + use super::*; + + #[pallet::pallet] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: frame_system::Config {} + + #[pallet::hooks] + impl Hooks> for Pallet { + // module hooks. + // one with block number arg and one without + fn on_initialize(_: BlockNumberFor) -> Weight { + assert!( + !MockedSystemCallbacks::pre_inherent_called(), + "Pre inherent hook goes after on_initialize" + ); + + Weight::from_parts(0, 0) + } + + fn on_idle(_: BlockNumberFor, _: Weight) -> Weight { + assert!( + MockedSystemCallbacks::post_transactions_called(), + "Post transactions hook goes before after on_idle" + ); + Weight::from_parts(0, 0) + } + + fn on_finalize(_: BlockNumberFor) { + assert!( + MockedSystemCallbacks::post_transactions_called(), + "Post transactions hook goes before after on_finalize" + ); + } + + fn on_runtime_upgrade() -> Weight { + sp_io::storage::set(super::TEST_KEY, "module".as_bytes()); + Weight::from_parts(0, 0) + } + } + + #[pallet::call] + impl Pallet { + pub fn allowed_unsigned(origin: OriginFor) -> DispatchResult { + frame_system::ensure_root(origin)?; + Ok(()) + } + + pub fn some_call(_: OriginFor) -> DispatchResult { + assert!(MockedSystemCallbacks::post_inherent_called()); + assert!(!MockedSystemCallbacks::post_transactions_called()); + assert!(System::inherents_applied()); + + Ok(()) + } + + #[pallet::weight({0})] + pub fn optional_inherent(origin: OriginFor) -> DispatchResult { + frame_system::ensure_none(origin)?; + + assert!(MockedSystemCallbacks::pre_inherent_called()); + assert!(!MockedSystemCallbacks::post_inherent_called(), "Should not already be called"); + assert!(!System::inherents_applied()); + + Ok(()) + } + + #[pallet::weight((0, DispatchClass::Mandatory))] + pub fn inherent(origin: OriginFor) -> DispatchResult { + frame_system::ensure_none(origin)?; + + assert!(MockedSystemCallbacks::pre_inherent_called()); + assert!(!MockedSystemCallbacks::post_inherent_called(), "Should not already be called"); + assert!(!System::inherents_applied()); + + Ok(()) + } + } + + #[pallet::inherent] + impl ProvideInherent for Pallet { + type Call = Call; + + type Error = sp_inherents::MakeFatalError<()>; + + const INHERENT_IDENTIFIER: [u8; 8] = *b"test1235"; + + fn create_inherent(_data: &InherentData) -> Option { + None + } + + fn is_inherent(call: &Self::Call) -> bool { + matches!(call, Call::::inherent {} | Call::::optional_inherent {}) + } + } + + #[pallet::validate_unsigned] + impl ValidateUnsigned for Pallet { + type Call = Call; + + // Inherent call is accepted for being dispatched + fn pre_dispatch(call: &Self::Call) -> Result<(), TransactionValidityError> { + match call { + Call::allowed_unsigned { .. } | + Call::optional_inherent { .. } | + Call::inherent { .. } => Ok(()), + _ => Err(UnknownTransaction::NoUnsignedValidator.into()), + } + } + + // Inherent call is not validated as unsigned + fn validate_unsigned(_source: TransactionSource, call: &Self::Call) -> TransactionValidity { + match call { + Call::allowed_unsigned { .. } => Ok(Default::default()), + _ => UnknownTransaction::NoUnsignedValidator.into(), + } + } + } +} + +frame_support::construct_runtime!( + pub struct Runtime + { + System: frame_system::{Pallet, Call, Config, Storage, Event}, + Balances: pallet_balances::{Pallet, Call, Storage, Config, Event}, + TransactionPayment: pallet_transaction_payment::{Pallet, Storage, Event}, + Custom: custom::{Pallet, Call, ValidateUnsigned, Inherent}, + Custom2: custom2::{Pallet, Call, ValidateUnsigned, Inherent}, + } +); + +parameter_types! { + pub BlockWeights: frame_system::limits::BlockWeights = + frame_system::limits::BlockWeights::builder() + .base_block(Weight::from_parts(10, 0)) + .for_class(DispatchClass::all(), |weights| weights.base_extrinsic = Weight::from_parts(5, 0)) + .for_class(DispatchClass::non_mandatory(), |weights| weights.max_total = Weight::from_parts(1024, u64::MAX).into()) + .build_or_panic(); + pub const DbWeight: RuntimeDbWeight = RuntimeDbWeight { + read: 10, + write: 100, + }; +} + +#[derive_impl(frame_system::config_preludes::TestDefaultConfig as frame_system::DefaultConfig)] +impl frame_system::Config for Runtime { + type BlockWeights = BlockWeights; + type RuntimeOrigin = RuntimeOrigin; + type Nonce = u64; + type RuntimeCall = RuntimeCall; + type Block = TestBlock; + type RuntimeEvent = RuntimeEvent; + type Version = RuntimeVersion; + type AccountData = pallet_balances::AccountData; + type PreInherents = MockedSystemCallbacks; + type PostInherents = MockedSystemCallbacks; + type PostTransactions = MockedSystemCallbacks; + type MultiBlockMigrator = MockedModeGetter; +} + +type Balance = u64; + +#[derive_impl(pallet_balances::config_preludes::TestDefaultConfig as pallet_balances::DefaultConfig)] +impl pallet_balances::Config for Runtime { + type Balance = Balance; + type AccountStore = System; +} + +parameter_types! { + pub const TransactionByteFee: Balance = 0; +} +impl pallet_transaction_payment::Config for Runtime { + type RuntimeEvent = RuntimeEvent; + type OnChargeTransaction = CurrencyAdapter; + type OperationalFeeMultiplier = ConstU8<5>; + type WeightToFee = IdentityFee; + type LengthToFee = ConstantMultiplier; + type FeeMultiplierUpdate = (); +} + +impl custom::Config for Runtime {} +impl custom2::Config for Runtime {} + +pub struct RuntimeVersion; +impl frame_support::traits::Get for RuntimeVersion { + fn get() -> sp_version::RuntimeVersion { + RuntimeVersionTestValues::get().clone() + } +} + +parameter_types! { + pub static RuntimeVersionTestValues: sp_version::RuntimeVersion = + Default::default(); +} + +type SignedExtra = ( + frame_system::CheckEra, + frame_system::CheckNonce, + frame_system::CheckWeight, + pallet_transaction_payment::ChargeTransactionPayment, +); +type TestXt = sp_runtime::testing::TestXt; +type TestBlock = Block; + +// Will contain `true` when the custom runtime logic was called. +const CUSTOM_ON_RUNTIME_KEY: &[u8] = b":custom:on_runtime"; + +struct CustomOnRuntimeUpgrade; +impl OnRuntimeUpgrade for CustomOnRuntimeUpgrade { + fn on_runtime_upgrade() -> Weight { + sp_io::storage::set(TEST_KEY, "custom_upgrade".as_bytes()); + sp_io::storage::set(CUSTOM_ON_RUNTIME_KEY, &true.encode()); + System::deposit_event(frame_system::Event::CodeUpdated); + + assert_eq!(0, System::last_runtime_upgrade_spec_version()); + + Weight::from_parts(100, 0) + } +} + +type Executive = super::Executive< + Runtime, + Block, + ChainContext, + Runtime, + AllPalletsWithSystem, + CustomOnRuntimeUpgrade, +>; + +parameter_types! { + pub static SystemCallbacksCalled: u32 = 0; +} + +pub struct MockedSystemCallbacks; +impl PreInherents for MockedSystemCallbacks { + fn pre_inherents() { + assert_eq!(SystemCallbacksCalled::get(), 0); + SystemCallbacksCalled::set(1); + // Change the storage to modify the root hash: + frame_support::storage::unhashed::put(b":pre_inherent", b"0"); + } +} + +impl PostInherents for MockedSystemCallbacks { + fn post_inherents() { + assert_eq!(SystemCallbacksCalled::get(), 1); + SystemCallbacksCalled::set(2); + // Change the storage to modify the root hash: + frame_support::storage::unhashed::put(b":post_inherent", b"0"); + } +} + +impl PostTransactions for MockedSystemCallbacks { + fn post_transactions() { + assert_eq!(SystemCallbacksCalled::get(), 2); + SystemCallbacksCalled::set(3); + // Change the storage to modify the root hash: + frame_support::storage::unhashed::put(b":post_transaction", b"0"); + } +} + +impl MockedSystemCallbacks { + fn pre_inherent_called() -> bool { + SystemCallbacksCalled::get() >= 1 + } + + fn post_inherent_called() -> bool { + SystemCallbacksCalled::get() >= 2 + } + + fn post_transactions_called() -> bool { + SystemCallbacksCalled::get() >= 3 + } + + fn reset() { + SystemCallbacksCalled::set(0); + frame_support::storage::unhashed::kill(b":pre_inherent"); + frame_support::storage::unhashed::kill(b":post_inherent"); + frame_support::storage::unhashed::kill(b":post_transaction"); + } +} + +parameter_types! { + pub static MbmActive: bool = false; +} + +pub struct MockedModeGetter; +impl MultiStepMigrator for MockedModeGetter { + fn ongoing() -> bool { + MbmActive::get() + } + + fn step() -> Weight { + Weight::zero() + } +} + +fn extra(nonce: u64, fee: Balance) -> SignedExtra { + ( + frame_system::CheckEra::from(Era::Immortal), + frame_system::CheckNonce::from(nonce), + frame_system::CheckWeight::new(), + pallet_transaction_payment::ChargeTransactionPayment::from(fee), + ) +} + +fn sign_extra(who: u64, nonce: u64, fee: Balance) -> Option<(u64, SignedExtra)> { + Some((who, extra(nonce, fee))) +} + +fn call_transfer(dest: u64, value: u64) -> RuntimeCall { + RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest, value }) +} + +#[test] +fn balance_transfer_dispatch_works() { + let mut t = frame_system::GenesisConfig::::default().build_storage().unwrap(); + pallet_balances::GenesisConfig:: { balances: vec![(1, 211)] } + .assimilate_storage(&mut t) + .unwrap(); + let xt = TestXt::new(call_transfer(2, 69), sign_extra(1, 0, 0)); + let weight = xt.get_dispatch_info().weight + + ::BlockWeights::get() + .get(DispatchClass::Normal) + .base_extrinsic; + let fee: Balance = + ::WeightToFee::weight_to_fee(&weight); + let mut t = sp_io::TestExternalities::new(t); + t.execute_with(|| { + Executive::initialize_block(&Header::new_from_number(1)); + let r = Executive::apply_extrinsic(xt); + assert!(r.is_ok()); + assert_eq!(>::total_balance(&1), 142 - fee); + assert_eq!(>::total_balance(&2), 69); + }); +} + +fn new_test_ext(balance_factor: Balance) -> sp_io::TestExternalities { + let mut t = frame_system::GenesisConfig::::default().build_storage().unwrap(); + pallet_balances::GenesisConfig:: { balances: vec![(1, 111 * balance_factor)] } + .assimilate_storage(&mut t) + .unwrap(); + let mut ext: sp_io::TestExternalities = t.into(); + ext.execute_with(|| { + SystemCallbacksCalled::set(0); + }); + ext +} + +fn new_test_ext_v0(balance_factor: Balance) -> sp_io::TestExternalities { + let mut t = frame_system::GenesisConfig::::default().build_storage().unwrap(); + pallet_balances::GenesisConfig:: { balances: vec![(1, 111 * balance_factor)] } + .assimilate_storage(&mut t) + .unwrap(); + (t, sp_runtime::StateVersion::V0).into() +} + +#[test] +fn block_import_works() { + block_import_works_inner( + new_test_ext_v0(1), + array_bytes::hex_n_into_unchecked( + "4826d3bdf87dbbc883d2ab274cbe272f58ed94a904619b59953e48294d1142d2", + ), + ); + block_import_works_inner( + new_test_ext(1), + array_bytes::hex_n_into_unchecked( + "d6b465f5a50c9f8d5a6edc0f01d285a6b19030f097d3aaf1649b7be81649f118", + ), + ); +} +fn block_import_works_inner(mut ext: sp_io::TestExternalities, state_root: H256) { + ext.execute_with(|| { + Executive::execute_block(Block { + header: Header { + parent_hash: [69u8; 32].into(), + number: 1, + state_root, + extrinsics_root: array_bytes::hex_n_into_unchecked( + "03170a2e7597b7b7e3d84c05391d139a62b157e78786d8c082f29dcf4c111314", + ), + digest: Digest { logs: vec![] }, + }, + extrinsics: vec![], + }); + }); +} + +#[test] +#[should_panic] +fn block_import_of_bad_state_root_fails() { + new_test_ext(1).execute_with(|| { + Executive::execute_block(Block { + header: Header { + parent_hash: [69u8; 32].into(), + number: 1, + state_root: [0u8; 32].into(), + extrinsics_root: array_bytes::hex_n_into_unchecked( + "03170a2e7597b7b7e3d84c05391d139a62b157e78786d8c082f29dcf4c111314", + ), + digest: Digest { logs: vec![] }, + }, + extrinsics: vec![], + }); + }); +} + +#[test] +#[should_panic] +fn block_import_of_bad_extrinsic_root_fails() { + new_test_ext(1).execute_with(|| { + Executive::execute_block(Block { + header: Header { + parent_hash: [69u8; 32].into(), + number: 1, + state_root: array_bytes::hex_n_into_unchecked( + "75e7d8f360d375bbe91bcf8019c01ab6362448b4a89e3b329717eb9d910340e5", + ), + extrinsics_root: [0u8; 32].into(), + digest: Digest { logs: vec![] }, + }, + extrinsics: vec![], + }); + }); +} + +#[test] +fn bad_extrinsic_not_inserted() { + let mut t = new_test_ext(1); + // bad nonce check! + let xt = TestXt::new(call_transfer(33, 69), sign_extra(1, 30, 0)); + t.execute_with(|| { + Executive::initialize_block(&Header::new_from_number(1)); + assert_err!( + Executive::apply_extrinsic(xt), + TransactionValidityError::Invalid(InvalidTransaction::Future) + ); + assert_eq!(>::extrinsic_index(), Some(0)); + }); +} + +#[test] +fn block_weight_limit_enforced() { + let mut t = new_test_ext(10000); + // given: TestXt uses the encoded len as fixed Len: + let xt = TestXt::new( + RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest: 33, value: 0 }), + sign_extra(1, 0, 0), + ); + let encoded = xt.encode(); + let encoded_len = encoded.len() as u64; + // on_initialize weight + base block execution weight + let block_weights = ::BlockWeights::get(); + let base_block_weight = Weight::from_parts(175, 0) + block_weights.base_block; + let limit = block_weights.get(DispatchClass::Normal).max_total.unwrap() - base_block_weight; + let num_to_exhaust_block = limit.ref_time() / (encoded_len + 5); + t.execute_with(|| { + Executive::initialize_block(&Header::new_from_number(1)); + // Base block execution weight + `on_initialize` weight from the custom module. + assert_eq!(>::block_weight().total(), base_block_weight); + + for nonce in 0..=num_to_exhaust_block { + let xt = TestXt::new( + RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest: 33, value: 0 }), + sign_extra(1, nonce.into(), 0), + ); + let res = Executive::apply_extrinsic(xt); + if nonce != num_to_exhaust_block { + assert!(res.is_ok()); + assert_eq!( + >::block_weight().total(), + //--------------------- on_initialize + block_execution + extrinsic_base weight + Weight::from_parts((encoded_len + 5) * (nonce + 1), 0) + base_block_weight, + ); + assert_eq!( + >::extrinsic_index(), + Some(nonce as u32 + 1) + ); + } else { + assert_eq!(res, Err(InvalidTransaction::ExhaustsResources.into())); + } + } + }); +} + +#[test] +fn block_weight_and_size_is_stored_per_tx() { + let xt = TestXt::new( + RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest: 33, value: 0 }), + sign_extra(1, 0, 0), + ); + let x1 = TestXt::new( + RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest: 33, value: 0 }), + sign_extra(1, 1, 0), + ); + let x2 = TestXt::new( + RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest: 33, value: 0 }), + sign_extra(1, 2, 0), + ); + let len = xt.clone().encode().len() as u32; + let mut t = new_test_ext(1); + t.execute_with(|| { + // Block execution weight + on_initialize weight from custom module + let base_block_weight = Weight::from_parts(175, 0) + + ::BlockWeights::get().base_block; + + Executive::initialize_block(&Header::new_from_number(1)); + + assert_eq!(>::block_weight().total(), base_block_weight); + assert_eq!(>::all_extrinsics_len(), 0); + + assert!(Executive::apply_extrinsic(xt.clone()).unwrap().is_ok()); + assert!(Executive::apply_extrinsic(x1.clone()).unwrap().is_ok()); + assert!(Executive::apply_extrinsic(x2.clone()).unwrap().is_ok()); + + // default weight for `TestXt` == encoded length. + let extrinsic_weight = Weight::from_parts(len as u64, 0) + + ::BlockWeights::get() + .get(DispatchClass::Normal) + .base_extrinsic; + assert_eq!( + >::block_weight().total(), + base_block_weight + 3u64 * extrinsic_weight, + ); + assert_eq!(>::all_extrinsics_len(), 3 * len); + + let _ = >::finalize(); + // All extrinsics length cleaned on `System::finalize` + assert_eq!(>::all_extrinsics_len(), 0); + + // Reset to a new block. + SystemCallbacksCalled::take(); + Executive::initialize_block(&Header::new_from_number(2)); + + // Block weight cleaned up on `System::initialize` + assert_eq!(>::block_weight().total(), base_block_weight); + }); +} + +#[test] +fn validate_unsigned() { + let valid = TestXt::new(RuntimeCall::Custom(custom::Call::allowed_unsigned {}), None); + let invalid = TestXt::new(RuntimeCall::Custom(custom::Call::unallowed_unsigned {}), None); + let mut t = new_test_ext(1); + + t.execute_with(|| { + assert_eq!( + Executive::validate_transaction( + TransactionSource::InBlock, + valid.clone(), + Default::default(), + ), + Ok(ValidTransaction::default()), + ); + assert_eq!( + Executive::validate_transaction( + TransactionSource::InBlock, + invalid.clone(), + Default::default(), + ), + Err(TransactionValidityError::Unknown(UnknownTransaction::NoUnsignedValidator)), + ); + // Need to initialize the block before applying extrinsics for the `MockedSystemCallbacks` + // check. + Executive::initialize_block(&Header::new_from_number(1)); + assert_eq!(Executive::apply_extrinsic(valid), Ok(Err(DispatchError::BadOrigin))); + assert_eq!( + Executive::apply_extrinsic(invalid), + Err(TransactionValidityError::Unknown(UnknownTransaction::NoUnsignedValidator)) + ); + }); +} + +#[test] +fn can_not_pay_for_tx_fee_on_full_lock() { + let mut t = new_test_ext(1); + t.execute_with(|| { + as fungible::MutateFreeze>::set_freeze(&(), &1, 110) + .unwrap(); + let xt = TestXt::new( + RuntimeCall::System(frame_system::Call::remark { remark: vec![1u8] }), + sign_extra(1, 0, 0), + ); + Executive::initialize_block(&Header::new_from_number(1)); + + assert_eq!(Executive::apply_extrinsic(xt), Err(InvalidTransaction::Payment.into()),); + assert_eq!(>::total_balance(&1), 111); + }); +} + +#[test] +fn block_hooks_weight_is_stored() { + new_test_ext(1).execute_with(|| { + Executive::initialize_block(&Header::new_from_number(1)); + Executive::finalize_block(); + // NOTE: might need updates over time if new weights are introduced. + // For now it only accounts for the base block execution weight and + // the `on_initialize` weight defined in the custom test module. + assert_eq!( + >::block_weight().total(), + Weight::from_parts(175 + 175 + 10, 0) + ); + }) +} + +#[test] +fn runtime_upgraded_should_work() { + new_test_ext(1).execute_with(|| { + RuntimeVersionTestValues::mutate(|v| *v = Default::default()); + // It should be added at genesis + assert!(LastRuntimeUpgrade::::exists()); + assert!(!Executive::runtime_upgraded()); + + RuntimeVersionTestValues::mutate(|v| { + *v = sp_version::RuntimeVersion { spec_version: 1, ..Default::default() } + }); + assert!(Executive::runtime_upgraded()); + + RuntimeVersionTestValues::mutate(|v| { + *v = sp_version::RuntimeVersion { + spec_version: 1, + spec_name: "test".into(), + ..Default::default() + } + }); + assert!(Executive::runtime_upgraded()); + + RuntimeVersionTestValues::mutate(|v| { + *v = sp_version::RuntimeVersion { + spec_version: 0, + impl_version: 2, + ..Default::default() + } + }); + assert!(!Executive::runtime_upgraded()); + + LastRuntimeUpgrade::::take(); + assert!(Executive::runtime_upgraded()); + }) +} + +#[test] +fn last_runtime_upgrade_was_upgraded_works() { + let test_data = vec![ + (0, "", 1, "", true), + (1, "", 1, "", false), + (1, "", 1, "test", true), + (1, "", 0, "", false), + (1, "", 0, "test", true), + ]; + + for (spec_version, spec_name, c_spec_version, c_spec_name, result) in test_data { + let current = sp_version::RuntimeVersion { + spec_version: c_spec_version, + spec_name: c_spec_name.into(), + ..Default::default() + }; + + let last = LastRuntimeUpgradeInfo { + spec_version: spec_version.into(), + spec_name: spec_name.into(), + }; + + assert_eq!(result, last.was_upgraded(¤t)); + } +} + +#[test] +fn custom_runtime_upgrade_is_called_before_modules() { + new_test_ext(1).execute_with(|| { + // Make sure `on_runtime_upgrade` is called. + RuntimeVersionTestValues::mutate(|v| { + *v = sp_version::RuntimeVersion { spec_version: 1, ..Default::default() } + }); + + Executive::initialize_block(&Header::new_from_number(1)); + + assert_eq!(&sp_io::storage::get(TEST_KEY).unwrap()[..], *b"module"); + assert_eq!(sp_io::storage::get(CUSTOM_ON_RUNTIME_KEY).unwrap(), true.encode()); + assert_eq!( + Some(RuntimeVersionTestValues::get().into()), + LastRuntimeUpgrade::::get(), + ) + }); +} + +#[test] +fn event_from_runtime_upgrade_is_included() { + new_test_ext(1).execute_with(|| { + // Make sure `on_runtime_upgrade` is called. + RuntimeVersionTestValues::mutate(|v| { + *v = sp_version::RuntimeVersion { spec_version: 1, ..Default::default() } + }); + + // set block number to non zero so events are not excluded + System::set_block_number(1); + + Executive::initialize_block(&Header::new_from_number(2)); + System::assert_last_event(frame_system::Event::::CodeUpdated.into()); + }); +} + +/// Regression test that ensures that the custom on runtime upgrade is called when executive is +/// used through the `ExecuteBlock` trait. +#[test] +fn custom_runtime_upgrade_is_called_when_using_execute_block_trait() { + let xt = TestXt::new( + RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest: 33, value: 0 }), + sign_extra(1, 0, 0), + ); + + let header = new_test_ext(1).execute_with(|| { + // Make sure `on_runtime_upgrade` is called. + RuntimeVersionTestValues::mutate(|v| { + *v = sp_version::RuntimeVersion { spec_version: 1, ..Default::default() } + }); + + // Let's build some fake block. + Executive::initialize_block(&Header::new_from_number(1)); + + Executive::apply_extrinsic(xt.clone()).unwrap().unwrap(); + + Executive::finalize_block() + }); + + // Reset to get the correct new genesis below. + RuntimeVersionTestValues::mutate(|v| { + *v = sp_version::RuntimeVersion { spec_version: 0, ..Default::default() } + }); + + new_test_ext(1).execute_with(|| { + // Make sure `on_runtime_upgrade` is called. + RuntimeVersionTestValues::mutate(|v| { + *v = sp_version::RuntimeVersion { spec_version: 1, ..Default::default() } + }); + + >>::execute_block(Block::new(header, vec![xt])); + + assert_eq!(&sp_io::storage::get(TEST_KEY).unwrap()[..], *b"module"); + assert_eq!(sp_io::storage::get(CUSTOM_ON_RUNTIME_KEY).unwrap(), true.encode()); + }); +} + +#[test] +fn all_weights_are_recorded_correctly() { + // Reset to get the correct new genesis below. + RuntimeVersionTestValues::take(); + + new_test_ext(1).execute_with(|| { + // Make sure `on_runtime_upgrade` is called for maximum complexity + RuntimeVersionTestValues::mutate(|v| { + *v = sp_version::RuntimeVersion { spec_version: 1, ..Default::default() } + }); + + let block_number = 1; + + Executive::initialize_block(&Header::new_from_number(block_number)); + + // Reset the last runtime upgrade info, to make the second call to `on_runtime_upgrade` + // succeed. + LastRuntimeUpgrade::::take(); + MockedSystemCallbacks::reset(); + + // All weights that show up in the `initialize_block_impl` + let custom_runtime_upgrade_weight = CustomOnRuntimeUpgrade::on_runtime_upgrade(); + let runtime_upgrade_weight = + ::on_runtime_upgrade(); + let on_initialize_weight = + >::on_initialize(block_number); + let base_block_weight = ::BlockWeights::get().base_block; + + // Weights are recorded correctly + assert_eq!( + frame_system::Pallet::::block_weight().total(), + custom_runtime_upgrade_weight + + runtime_upgrade_weight + + on_initialize_weight + + base_block_weight, + ); + }); +} + +#[test] +fn offchain_worker_works_as_expected() { + new_test_ext(1).execute_with(|| { + let parent_hash = sp_core::H256::from([69u8; 32]); + let mut digest = Digest::default(); + digest.push(DigestItem::Seal([1, 2, 3, 4], vec![5, 6, 7, 8])); + + let header = Header::new(1, H256::default(), H256::default(), parent_hash, digest.clone()); + + Executive::offchain_worker(&header); + + assert_eq!(digest, System::digest()); + assert_eq!(parent_hash, System::block_hash(0)); + assert_eq!(header.hash(), System::block_hash(1)); + }); +} + +#[test] +fn calculating_storage_root_twice_works() { + let call = RuntimeCall::Custom(custom::Call::calculate_storage_root {}); + let xt = TestXt::new(call, sign_extra(1, 0, 0)); + + let header = new_test_ext(1).execute_with(|| { + // Let's build some fake block. + Executive::initialize_block(&Header::new_from_number(1)); + + Executive::apply_extrinsic(xt.clone()).unwrap().unwrap(); + + Executive::finalize_block() + }); + + new_test_ext(1).execute_with(|| { + Executive::execute_block(Block::new(header, vec![xt])); + }); +} + +#[test] +#[should_panic(expected = "Invalid inherent position for extrinsic at index 1")] +fn invalid_inherent_position_fail() { + let xt1 = TestXt::new( + RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest: 33, value: 0 }), + sign_extra(1, 0, 0), + ); + let xt2 = TestXt::new(RuntimeCall::Custom(custom::Call::inherent {}), None); + + let header = new_test_ext(1).execute_with(|| { + // Let's build some fake block. + Executive::initialize_block(&Header::new_from_number(1)); + + Executive::apply_extrinsic(xt1.clone()).unwrap().unwrap(); + Executive::apply_extrinsic(xt2.clone()).unwrap().unwrap(); + + Executive::finalize_block() + }); + + new_test_ext(1).execute_with(|| { + Executive::execute_block(Block::new(header, vec![xt1, xt2])); + }); +} + +#[test] +fn valid_inherents_position_works() { + let xt1 = TestXt::new(RuntimeCall::Custom(custom::Call::inherent {}), None); + let xt2 = TestXt::new(call_transfer(33, 0), sign_extra(1, 0, 0)); + + let header = new_test_ext(1).execute_with(|| { + // Let's build some fake block. + Executive::initialize_block(&Header::new_from_number(1)); + + Executive::apply_extrinsic(xt1.clone()).unwrap().unwrap(); + Executive::apply_extrinsic(xt2.clone()).unwrap().unwrap(); + + Executive::finalize_block() + }); + + new_test_ext(1).execute_with(|| { + Executive::execute_block(Block::new(header, vec![xt1, xt2])); + }); +} + +#[test] +#[should_panic(expected = "A call was labelled as mandatory, but resulted in an Error.")] +fn invalid_inherents_fail_block_execution() { + let xt1 = TestXt::new(RuntimeCall::Custom(custom::Call::inherent {}), sign_extra(1, 0, 0)); + + new_test_ext(1).execute_with(|| { + Executive::execute_block(Block::new( + Header::new(1, H256::default(), H256::default(), [69u8; 32].into(), Digest::default()), + vec![xt1], + )); + }); +} + +// Inherents are created by the runtime and don't need to be validated. +#[test] +fn inherents_fail_validate_block() { + let xt1 = TestXt::new(RuntimeCall::Custom(custom::Call::inherent {}), None); + + new_test_ext(1).execute_with(|| { + assert_eq!( + Executive::validate_transaction(TransactionSource::External, xt1, H256::random()) + .unwrap_err(), + InvalidTransaction::MandatoryValidation.into() + ); + }) +} + +/// Inherents still work while `initialize_block` forbids transactions. +#[test] +fn inherents_ok_while_exts_forbidden_works() { + let xt1 = TestXt::new(RuntimeCall::Custom(custom::Call::inherent {}), None); + + let header = new_test_ext(1).execute_with(|| { + Executive::initialize_block(&Header::new_from_number(1)); + + Executive::apply_extrinsic(xt1.clone()).unwrap().unwrap(); + // This is not applied: + Executive::finalize_block() + }); + + new_test_ext(1).execute_with(|| { + // Tell `initialize_block` to forbid extrinsics: + Executive::execute_block(Block::new(header, vec![xt1])); + }); +} + +/// Refuses to import blocks with transactions during `OnlyInherents` mode. +#[test] +#[should_panic = "Only inherents are allowed in this block"] +fn transactions_in_only_inherents_block_errors() { + let xt1 = TestXt::new(RuntimeCall::Custom(custom::Call::inherent {}), None); + let xt2 = TestXt::new(call_transfer(33, 0), sign_extra(1, 0, 0)); + + let header = new_test_ext(1).execute_with(|| { + Executive::initialize_block(&Header::new_from_number(1)); + + Executive::apply_extrinsic(xt1.clone()).unwrap().unwrap(); + Executive::apply_extrinsic(xt2.clone()).unwrap().unwrap(); + + Executive::finalize_block() + }); + + new_test_ext(1).execute_with(|| { + MbmActive::set(true); + Executive::execute_block(Block::new(header, vec![xt1, xt2])); + }); +} + +/// Same as above but no error. +#[test] +fn transactions_in_normal_block_works() { + let xt1 = TestXt::new(RuntimeCall::Custom(custom::Call::inherent {}), None); + let xt2 = TestXt::new(call_transfer(33, 0), sign_extra(1, 0, 0)); + + let header = new_test_ext(1).execute_with(|| { + Executive::initialize_block(&Header::new_from_number(1)); + + Executive::apply_extrinsic(xt1.clone()).unwrap().unwrap(); + Executive::apply_extrinsic(xt2.clone()).unwrap().unwrap(); + + Executive::finalize_block() + }); + + new_test_ext(1).execute_with(|| { + // Tell `initialize_block` to forbid extrinsics: + Executive::execute_block(Block::new(header, vec![xt1, xt2])); + }); +} + +#[test] +#[cfg(feature = "try-runtime")] +fn try_execute_block_works() { + let xt1 = TestXt::new(RuntimeCall::Custom(custom::Call::inherent {}), None); + let xt2 = TestXt::new(call_transfer(33, 0), sign_extra(1, 0, 0)); + + let header = new_test_ext(1).execute_with(|| { + Executive::initialize_block(&Header::new_from_number(1)); + + Executive::apply_extrinsic(xt1.clone()).unwrap().unwrap(); + Executive::apply_extrinsic(xt2.clone()).unwrap().unwrap(); + + Executive::finalize_block() + }); + + new_test_ext(1).execute_with(|| { + Executive::try_execute_block( + Block::new(header, vec![xt1, xt2]), + true, + true, + frame_try_runtime::TryStateSelect::All, + ) + .unwrap(); + }); +} + +/// Same as `extrinsic_while_exts_forbidden_errors` but using the try-runtime function. +#[test] +#[cfg(feature = "try-runtime")] +#[should_panic = "Only inherents allowed"] +fn try_execute_tx_forbidden_errors() { + let xt1 = TestXt::new(RuntimeCall::Custom(custom::Call::inherent {}), None); + let xt2 = TestXt::new(call_transfer(33, 0), sign_extra(1, 0, 0)); + + let header = new_test_ext(1).execute_with(|| { + // Let's build some fake block. + Executive::initialize_block(&Header::new_from_number(1)); + + Executive::apply_extrinsic(xt1.clone()).unwrap().unwrap(); + Executive::apply_extrinsic(xt2.clone()).unwrap().unwrap(); + + Executive::finalize_block() + }); + + new_test_ext(1).execute_with(|| { + MbmActive::set(true); + Executive::try_execute_block( + Block::new(header, vec![xt1, xt2]), + true, + true, + frame_try_runtime::TryStateSelect::All, + ) + .unwrap(); + }); +} + +/// Check that `ensure_inherents_are_first` reports the correct indices. +#[test] +fn ensure_inherents_are_first_works() { + let in1 = TestXt::new(RuntimeCall::Custom(custom::Call::inherent {}), None); + let in2 = TestXt::new(RuntimeCall::Custom2(custom2::Call::inherent {}), None); + let xt2 = TestXt::new(call_transfer(33, 0), sign_extra(1, 0, 0)); + + // Mocked empty header: + let header = new_test_ext(1).execute_with(|| { + Executive::initialize_block(&Header::new_from_number(1)); + Executive::finalize_block() + }); + + new_test_ext(1).execute_with(|| { + assert_ok!(Runtime::ensure_inherents_are_first(&Block::new(header.clone(), vec![]),), 0); + assert_ok!( + Runtime::ensure_inherents_are_first(&Block::new(header.clone(), vec![xt2.clone()]),), + 0 + ); + assert_ok!( + Runtime::ensure_inherents_are_first(&Block::new(header.clone(), vec![in1.clone()])), + 1 + ); + assert_ok!( + Runtime::ensure_inherents_are_first(&Block::new( + header.clone(), + vec![in1.clone(), xt2.clone()] + ),), + 1 + ); + assert_ok!( + Runtime::ensure_inherents_are_first(&Block::new( + header.clone(), + vec![in2.clone(), in1.clone(), xt2.clone()] + ),), + 2 + ); + + assert_eq!( + Runtime::ensure_inherents_are_first(&Block::new( + header.clone(), + vec![xt2.clone(), in1.clone()] + ),), + Err(1) + ); + assert_eq!( + Runtime::ensure_inherents_are_first(&Block::new( + header.clone(), + vec![xt2.clone(), xt2.clone(), in1.clone()] + ),), + Err(2) + ); + assert_eq!( + Runtime::ensure_inherents_are_first(&Block::new( + header.clone(), + vec![xt2.clone(), xt2.clone(), xt2.clone(), in2.clone()] + ),), + Err(3) + ); + }); +} + +/// Check that block execution rejects blocks with transactions in them while MBMs are active and +/// also that all the system callbacks are called correctly. +#[test] +fn callbacks_in_block_execution_works() { + callbacks_in_block_execution_works_inner(false); + callbacks_in_block_execution_works_inner(true); +} + +fn callbacks_in_block_execution_works_inner(mbms_active: bool) { + MbmActive::set(mbms_active); + + for (n_in, n_tx) in (0..10usize).zip(0..10usize) { + let mut extrinsics = Vec::new(); + + let header = new_test_ext(10).execute_with(|| { + MockedSystemCallbacks::reset(); + Executive::initialize_block(&Header::new_from_number(1)); + assert_eq!(SystemCallbacksCalled::get(), 1); + + for i in 0..n_in { + let xt = if i % 2 == 0 { + TestXt::new(RuntimeCall::Custom(custom::Call::inherent {}), None) + } else { + TestXt::new(RuntimeCall::Custom2(custom2::Call::optional_inherent {}), None) + }; + Executive::apply_extrinsic(xt.clone()).unwrap().unwrap(); + extrinsics.push(xt); + } + + for t in 0..n_tx { + let xt = TestXt::new( + RuntimeCall::Custom2(custom2::Call::some_call {}), + sign_extra(1, t as u64, 0), + ); + // Extrinsics can be applied even when MBMs are active. Only the `execute_block` + // will reject it. + Executive::apply_extrinsic(xt.clone()).unwrap().unwrap(); + extrinsics.push(xt); + } + + Executive::finalize_block() + }); + assert_eq!(SystemCallbacksCalled::get(), 3); + + new_test_ext(10).execute_with(|| { + let header = std::panic::catch_unwind(|| { + Executive::execute_block(Block::new(header, extrinsics)); + }); + + match header { + Err(e) => { + let err = e.downcast::<&str>().unwrap(); + assert_eq!(*err, "Only inherents are allowed in this block"); + assert!( + MbmActive::get() && n_tx > 0, + "Transactions should be rejected when MBMs are active" + ); + }, + Ok(_) => { + assert_eq!(SystemCallbacksCalled::get(), 3); + assert!( + !MbmActive::get() || n_tx == 0, + "MBMs should be deactivated after finalization" + ); + }, + } + }); + } +} + +#[test] +fn post_inherent_called_after_all_inherents() { + let in1 = TestXt::new(RuntimeCall::Custom2(custom2::Call::inherent {}), None); + let xt1 = TestXt::new(RuntimeCall::Custom2(custom2::Call::some_call {}), sign_extra(1, 0, 0)); + + let header = new_test_ext(1).execute_with(|| { + // Let's build some fake block. + Executive::initialize_block(&Header::new_from_number(1)); + + Executive::apply_extrinsic(in1.clone()).unwrap().unwrap(); + Executive::apply_extrinsic(xt1.clone()).unwrap().unwrap(); + + Executive::finalize_block() + }); + + #[cfg(feature = "try-runtime")] + new_test_ext(1).execute_with(|| { + Executive::try_execute_block( + Block::new(header.clone(), vec![in1.clone(), xt1.clone()]), + true, + true, + frame_try_runtime::TryStateSelect::All, + ) + .unwrap(); + assert!(MockedSystemCallbacks::post_transactions_called()); + }); + + new_test_ext(1).execute_with(|| { + MockedSystemCallbacks::reset(); + Executive::execute_block(Block::new(header, vec![in1, xt1])); + assert!(MockedSystemCallbacks::post_transactions_called()); + }); +} + +/// Regression test for AppSec finding #40. +#[test] +fn post_inherent_called_after_all_optional_inherents() { + let in1 = TestXt::new(RuntimeCall::Custom2(custom2::Call::optional_inherent {}), None); + let xt1 = TestXt::new(RuntimeCall::Custom2(custom2::Call::some_call {}), sign_extra(1, 0, 0)); + + let header = new_test_ext(1).execute_with(|| { + // Let's build some fake block. + Executive::initialize_block(&Header::new_from_number(1)); + + Executive::apply_extrinsic(in1.clone()).unwrap().unwrap(); + Executive::apply_extrinsic(xt1.clone()).unwrap().unwrap(); + + Executive::finalize_block() + }); + + #[cfg(feature = "try-runtime")] + new_test_ext(1).execute_with(|| { + Executive::try_execute_block( + Block::new(header.clone(), vec![in1.clone(), xt1.clone()]), + true, + true, + frame_try_runtime::TryStateSelect::All, + ) + .unwrap(); + assert!(MockedSystemCallbacks::post_transactions_called()); + }); + + new_test_ext(1).execute_with(|| { + MockedSystemCallbacks::reset(); + Executive::execute_block(Block::new(header, vec![in1, xt1])); + assert!(MockedSystemCallbacks::post_transactions_called()); + }); +} + +#[test] +fn is_inherent_works() { + let ext = TestXt::new(RuntimeCall::Custom2(custom2::Call::inherent {}), None); + assert!(Runtime::is_inherent(&ext)); + let ext = TestXt::new(RuntimeCall::Custom2(custom2::Call::optional_inherent {}), None); + assert!(Runtime::is_inherent(&ext)); + + let ext = TestXt::new(call_transfer(33, 0), sign_extra(1, 0, 0)); + assert!(!Runtime::is_inherent(&ext)); + + let ext = TestXt::new(RuntimeCall::Custom2(custom2::Call::allowed_unsigned {}), None); + assert!(!Runtime::is_inherent(&ext), "Unsigned ext are not automatically inherents"); +} diff --git a/substrate/frame/migrations/Cargo.toml b/substrate/frame/migrations/Cargo.toml new file mode 100644 index 000000000000..2914aa9ea973 --- /dev/null +++ b/substrate/frame/migrations/Cargo.toml @@ -0,0 +1,64 @@ +[package] +name = "pallet-migrations" +version = "1.0.0" +description = "FRAME pallet to execute multi-block migrations." +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +docify = "0.1.14" +impl-trait-for-tuples = "0.2.2" +log = "0.4.18" +scale-info = { version = "2.0.0", default-features = false, features = ["derive"] } + +frame-benchmarking = { default-features = false, optional = true, path = "../benchmarking" } +frame-support = { default-features = false, path = "../support" } +frame-system = { default-features = false, path = "../system" } +sp-core = { path = "../../primitives/core", default-features = false } +sp-std = { path = "../../primitives/std", default-features = false } +sp-runtime = { path = "../../primitives/runtime", default-features = false } + +[dev-dependencies] +frame-executive = { path = "../executive" } +sp-api = { path = "../../primitives/api", features = ["std"] } +sp-block-builder = { path = "../../primitives/block-builder", features = ["std"] } +sp-io = { path = "../../primitives/io", features = ["std"] } +sp-tracing = { path = "../../primitives/tracing", features = ["std"] } +sp-version = { path = "../../primitives/version", features = ["std"] } + +pretty_assertions = "1.3.0" + +[features] +default = ["std"] + +std = [ + "codec/std", + "frame-benchmarking?/std", + "frame-support/std", + "frame-system/std", + "log/std", + "scale-info/std", + "sp-core/std", + "sp-runtime/std", + "sp-std/std", +] + +runtime-benchmarks = [ + "frame-benchmarking/runtime-benchmarks", + "frame-support/runtime-benchmarks", + "frame-system/runtime-benchmarks", + "sp-runtime/runtime-benchmarks", +] + +try-runtime = [ + "frame-executive/try-runtime", + "frame-support/try-runtime", + "frame-system/try-runtime", + "sp-runtime/try-runtime", +] diff --git a/substrate/frame/migrations/src/benchmarking.rs b/substrate/frame/migrations/src/benchmarking.rs new file mode 100644 index 000000000000..8ad1fa50d149 --- /dev/null +++ b/substrate/frame/migrations/src/benchmarking.rs @@ -0,0 +1,222 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![cfg(feature = "runtime-benchmarks")] + +use super::*; + +use frame_benchmarking::{v2::*, BenchmarkError}; +use frame_system::{Pallet as System, RawOrigin}; +use sp_runtime::traits::One; + +fn assert_last_event(generic_event: ::RuntimeEvent) { + frame_system::Pallet::::assert_last_event(generic_event.into()); +} + +#[benchmarks] +mod benches { + use super::*; + use frame_support::traits::Hooks; + + #[benchmark] + fn onboard_new_mbms() { + T::Migrations::set_fail_after(0); // Should not be called anyway. + assert!(!Cursor::::exists()); + + #[block] + { + Pallet::::onboard_new_mbms(); + } + + assert_last_event::(Event::UpgradeStarted { migrations: 1 }.into()); + } + + #[benchmark] + fn progress_mbms_none() { + T::Migrations::set_fail_after(0); // Should not be called anyway. + assert!(!Cursor::::exists()); + + #[block] + { + Pallet::::progress_mbms(One::one()); + } + } + + /// All migrations completed. + #[benchmark] + fn exec_migration_completed() -> Result<(), BenchmarkError> { + T::Migrations::set_fail_after(0); // Should not be called anyway. + assert_eq!(T::Migrations::len(), 1, "Setup failed"); + let c = ActiveCursor { index: 1, inner_cursor: None, started_at: 0u32.into() }; + let mut meter = WeightMeter::with_limit(T::MaxServiceWeight::get()); + System::::set_block_number(1u32.into()); + + #[block] + { + Pallet::::exec_migration(c, false, &mut meter); + } + + assert_last_event::(Event::UpgradeCompleted {}.into()); + + Ok(()) + } + + /// No migration runs since it is skipped as historic. + #[benchmark] + fn exec_migration_skipped_historic() -> Result<(), BenchmarkError> { + T::Migrations::set_fail_after(0); // Should not be called anyway. + assert_eq!(T::Migrations::len(), 1, "Setup failed"); + let c = ActiveCursor { index: 0, inner_cursor: None, started_at: 0u32.into() }; + + let id: IdentifierOf = T::Migrations::nth_id(0).unwrap().try_into().unwrap(); + Historic::::insert(id, ()); + + let mut meter = WeightMeter::with_limit(T::MaxServiceWeight::get()); + System::::set_block_number(1u32.into()); + + #[block] + { + Pallet::::exec_migration(c, false, &mut meter); + } + + assert_last_event::(Event::MigrationSkipped { index: 0 }.into()); + + Ok(()) + } + + /// Advance a migration by one step. + #[benchmark] + fn exec_migration_advance() -> Result<(), BenchmarkError> { + T::Migrations::set_success_after(1); + assert_eq!(T::Migrations::len(), 1, "Setup failed"); + let c = ActiveCursor { index: 0, inner_cursor: None, started_at: 0u32.into() }; + let mut meter = WeightMeter::with_limit(T::MaxServiceWeight::get()); + System::::set_block_number(1u32.into()); + + #[block] + { + Pallet::::exec_migration(c, false, &mut meter); + } + + assert_last_event::(Event::MigrationAdvanced { index: 0, took: One::one() }.into()); + + Ok(()) + } + + /// Successfully complete a migration. + #[benchmark] + fn exec_migration_complete() -> Result<(), BenchmarkError> { + T::Migrations::set_success_after(0); + assert_eq!(T::Migrations::len(), 1, "Setup failed"); + let c = ActiveCursor { index: 0, inner_cursor: None, started_at: 0u32.into() }; + let mut meter = WeightMeter::with_limit(T::MaxServiceWeight::get()); + System::::set_block_number(1u32.into()); + + #[block] + { + Pallet::::exec_migration(c, false, &mut meter); + } + + assert_last_event::(Event::MigrationCompleted { index: 0, took: One::one() }.into()); + + Ok(()) + } + + #[benchmark] + fn exec_migration_fail() -> Result<(), BenchmarkError> { + T::Migrations::set_fail_after(0); + assert_eq!(T::Migrations::len(), 1, "Setup failed"); + let c = ActiveCursor { index: 0, inner_cursor: None, started_at: 0u32.into() }; + let mut meter = WeightMeter::with_limit(T::MaxServiceWeight::get()); + System::::set_block_number(1u32.into()); + + #[block] + { + Pallet::::exec_migration(c, false, &mut meter); + } + + assert_last_event::(Event::UpgradeFailed {}.into()); + + Ok(()) + } + + #[benchmark] + fn on_init_loop() { + T::Migrations::set_fail_after(0); // Should not be called anyway. + System::::set_block_number(1u32.into()); + Pallet::::on_runtime_upgrade(); + + #[block] + { + Pallet::::on_initialize(1u32.into()); + } + } + + #[benchmark] + fn force_set_cursor() { + #[extrinsic_call] + _(RawOrigin::Root, Some(cursor::())); + } + + #[benchmark] + fn force_set_active_cursor() { + #[extrinsic_call] + _(RawOrigin::Root, 0, None, None); + } + + #[benchmark] + fn force_onboard_mbms() { + #[extrinsic_call] + _(RawOrigin::Root); + } + + #[benchmark] + fn clear_historic(n: Linear<0, { DEFAULT_HISTORIC_BATCH_CLEAR_SIZE * 2 }>) { + let id_max_len = ::IdentifierMaxLen::get(); + assert!(id_max_len >= 4, "Precondition violated"); + + for i in 0..DEFAULT_HISTORIC_BATCH_CLEAR_SIZE * 2 { + let id = IdentifierOf::::truncate_from( + i.encode().into_iter().cycle().take(id_max_len as usize).collect::>(), + ); + + Historic::::insert(&id, ()); + } + + #[extrinsic_call] + _( + RawOrigin::Root, + HistoricCleanupSelector::Wildcard { limit: n.into(), previous_cursor: None }, + ); + } + + fn cursor() -> CursorOf { + // Note: The weight of a function can depend on the weight of reading the `inner_cursor`. + // `Cursor` is a user provided type. Now instead of requiring something like `Cursor: + // From`, we instead rely on the fact that it is MEL and the PoV benchmarking will + // therefore already take the MEL bound, even when the cursor in storage is `None`. + MigrationCursor::Active(ActiveCursor { + index: u32::MAX, + inner_cursor: None, + started_at: 0u32.into(), + }) + } + + // Implements a test for each benchmark. Execute with: + // `cargo test -p pallet-migrations --features runtime-benchmarks`. + impl_benchmark_test_suite!(Pallet, crate::mock::new_test_ext(), crate::mock::Test); +} diff --git a/substrate/frame/migrations/src/lib.rs b/substrate/frame/migrations/src/lib.rs new file mode 100644 index 000000000000..cd57d89f440f --- /dev/null +++ b/substrate/frame/migrations/src/lib.rs @@ -0,0 +1,746 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![deny(missing_docs)] +#![deny(rustdoc::broken_intra_doc_links)] + +//! # `pallet-migrations` +//! +//! Provides multi block migrations for FRAME runtimes. +//! +//! ## Overview +//! +//! The pallet takes care of executing a batch of multi-step migrations over multiple blocks. The +//! process starts on each runtime upgrade. Normal and operational transactions are paused while +//! migrations are on-going. +//! +//! ### Example +//! +//! This example demonstrates a simple mocked walk through of a basic success scenario. The pallet +//! is configured with two migrations: one succeeding after just one step, and the second one +//! succeeding after two steps. A runtime upgrade is then enacted and the block number is advanced +//! until all migrations finish executing. Afterwards, the recorded historic migrations are +//! checked and events are asserted. +#![doc = docify::embed!("substrate/frame/migrations/src/tests.rs", simple_works)] +//! +//! ## Pallet API +//! +//! See the [`pallet`] module for more information about the interfaces this pallet exposes, +//! including its configuration trait, dispatchables, storage items, events and errors. +//! +//! Otherwise noteworthy API of this pallet include its implementation of the +//! [`MultiStepMigrator`] trait. This must be plugged into +//! [`frame_system::Config::MultiBlockMigrator`] for proper function. +//! +//! The API contains some calls for emergency management. They are all prefixed with `force_` and +//! should normally not be needed. Pay special attention prior to using them. +//! +//! ### Design Goals +//! +//! 1. Must automatically execute migrations over multiple blocks. +//! 2. Must expose information about whether migrations are ongoing. +//! 3. Must respect pessimistic weight bounds of migrations. +//! 4. Must execute migrations in order. Skipping is not allowed; migrations are run on a +//! all-or-nothing basis. +//! 5. Must prevent re-execution of past migrations. +//! 6. Must provide transactional storage semantics for migrations. +//! 7. Must guarantee progress. +//! +//! ### Design +//! +//! Migrations are provided to the pallet through the associated type [`Config::Migrations`] of type +//! [`SteppedMigrations`]. This allows multiple migrations to be aggregated through a tuple. It +//! simplifies the trait bounds since all associated types of the trait must be provided by the +//! pallet. The actual progress of the pallet is stored in the [`Cursor`] storage item. This can +//! either be [`MigrationCursor::Active`] or [`MigrationCursor::Stuck`]. In the active case it +//! points to the currently active migration and stores its inner cursor. The inner cursor can then +//! be used by the migration to store its inner state and advance. Each time when the migration +//! returns `Some(cursor)`, it signals the pallet that it is not done yet. +//! The cursor is reset on each runtime upgrade. This ensures that it starts to execute at the +//! first migration in the vector. The pallets cursor is only ever incremented or set to `Stuck` +//! once it encounters an error (Goal 4). Once in the stuck state, the pallet will stay stuck until +//! it is fixed through manual governance intervention. +//! As soon as the cursor of the pallet becomes `Some(_)`; [`MultiStepMigrator::ongoing`] returns +//! `true` (Goal 2). This can be used by upstream code to possibly pause transactions. +//! In `on_initialize` the pallet will load the current migration and check whether it was already +//! executed in the past by checking for membership of its ID in the [`Historic`] set. Historic +//! migrations are skipped without causing an error. Each successfully executed migration is added +//! to this set (Goal 5). +//! This proceeds until no more migrations remain. At that point, the event `UpgradeCompleted` is +//! emitted (Goal 1). +//! The execution of each migration happens by calling [`SteppedMigration::transactional_step`]. +//! This function wraps the inner `step` function into a transactional layer to allow rollback in +//! the error case (Goal 6). +//! Weight limits must be checked by the migration itself. The pallet provides a [`WeightMeter`] for +//! that purpose. The pallet may return [`SteppedMigrationError::InsufficientWeight`] at any point. +//! In that scenario, one of two things will happen: if that migration was exclusively executed +//! in this block, and therefore required more than the maximum amount of weight possible, the +//! process becomes `Stuck`. Otherwise, one re-attempt is executed with the same logic in the next +//! block (Goal 3). Progress through the migrations is guaranteed by providing a timeout for each +//! migration via [`SteppedMigration::max_steps`]. The pallet **ONLY** guarantees progress if this +//! is set to sensible limits (Goal 7). +//! +//! ### Scenario: Governance cleanup +//! +//! Every now and then, governance can make use of the [`clear_historic`][Pallet::clear_historic] +//! call. This ensures that no old migrations pile up in the [`Historic`] set. This can be done very +//! rarely, since the storage should not grow quickly and the lookup weight does not suffer much. +//! Another possibility would be to have a synchronous single-block migration perpetually deployed +//! that cleans them up before the MBMs start. +//! +//! ### Scenario: Successful upgrade +//! +//! The standard procedure for a successful runtime upgrade can look like this: +//! 1. Migrations are configured in the `Migrations` config item. All migrations expose +//! [`max_steps`][SteppedMigration::max_steps], are error tolerant, check their weight bounds and +//! have a unique identifier. +//! 2. The runtime upgrade is enacted. An `UpgradeStarted` event is +//! followed by lots of `MigrationAdvanced` and `MigrationCompleted` events. Finally +//! `UpgradeCompleted` is emitted. +//! 3. Cleanup as described in the governance scenario be executed at any time after the migrations +//! completed. +//! +//! ### Advice: Failed upgrades +//! +//! Failed upgrades cannot be recovered from automatically and require governance intervention. Set +//! up monitoring for `UpgradeFailed` events to be made aware of any failures. The hook +//! [`FailedMigrationHandler::failed`] should be setup in a way that it allows governance to act, +//! but still prevent other transactions from interacting with the inconsistent storage state. Note +//! that this is paramount, since the inconsistent state might contain a faulty balance amount or +//! similar that could cause great harm if user transactions don't remain suspended. One way to +//! implement this would be to use the `SafeMode` or `TxPause` pallets that can prevent most user +//! interactions but still allow a whitelisted set of governance calls. +//! +//! ### Remark: Failed migrations +//! +//! Failed migrations are not added to the `Historic` set. This means that an erroneous +//! migration must be removed and fixed manually. This already applies, even before considering the +//! historic set. +//! +//! ### Remark: Transactional processing +//! +//! You can see the transactional semantics for migration steps as mostly useless, since in the +//! stuck case the state is already messed up. This just prevents it from becoming even more messed +//! up, but doesn't prevent it in the first place. + +#![cfg_attr(not(feature = "std"), no_std)] + +mod benchmarking; +mod mock; +pub mod mock_helpers; +mod tests; +pub mod weights; + +pub use pallet::*; +pub use weights::WeightInfo; + +use codec::{Decode, Encode, MaxEncodedLen}; +use core::ops::ControlFlow; +use frame_support::{ + defensive, defensive_assert, + migrations::*, + traits::Get, + weights::{Weight, WeightMeter}, + BoundedVec, +}; +use frame_system::{pallet_prelude::BlockNumberFor, Pallet as System}; +use sp_runtime::Saturating; +use sp_std::vec::Vec; + +/// Points to the next migration to execute. +#[derive(Debug, Clone, Eq, PartialEq, Encode, Decode, scale_info::TypeInfo, MaxEncodedLen)] +pub enum MigrationCursor { + /// Points to the currently active migration and its inner cursor. + Active(ActiveCursor), + + /// Migration got stuck and cannot proceed. This is bad. + Stuck, +} + +impl MigrationCursor { + /// Try to return self as an [`ActiveCursor`]. + pub fn as_active(&self) -> Option<&ActiveCursor> { + match self { + MigrationCursor::Active(active) => Some(active), + MigrationCursor::Stuck => None, + } + } +} + +impl From> + for MigrationCursor +{ + fn from(active: ActiveCursor) -> Self { + MigrationCursor::Active(active) + } +} + +/// Points to the currently active migration and its inner cursor. +#[derive(Debug, Clone, Eq, PartialEq, Encode, Decode, scale_info::TypeInfo, MaxEncodedLen)] +pub struct ActiveCursor { + /// The index of the migration in the MBM tuple. + pub index: u32, + /// The cursor of the migration that is referenced by `index`. + pub inner_cursor: Option, + /// The block number that the migration started at. + /// + /// This is used to calculate how many blocks it took. + pub started_at: BlockNumber, +} + +impl ActiveCursor { + /// Advance the overarching cursor to the next migration. + pub(crate) fn goto_next_migration(&mut self, current_block: BlockNumber) { + self.index.saturating_inc(); + self.inner_cursor = None; + self.started_at = current_block; + } +} + +/// How to clear the records of historic migrations. +#[derive(Debug, Clone, Eq, PartialEq, Encode, Decode, scale_info::TypeInfo)] +pub enum HistoricCleanupSelector { + /// Clear exactly these entries. + /// + /// This is the advised way of doing it. + Specific(Vec), + + /// Clear up to this many entries + Wildcard { + /// How many should be cleared in this call at most. + limit: Option, + /// The cursor that was emitted from any previous `HistoricCleared`. + /// + /// Does not need to be passed when clearing the first batch. + previous_cursor: Option>, + }, +} + +/// The default number of entries that should be cleared by a `HistoricCleanupSelector::Wildcard`. +/// +/// The caller can explicitly specify a higher amount. Benchmarks are run with twice this value. +const DEFAULT_HISTORIC_BATCH_CLEAR_SIZE: u32 = 128; + +impl HistoricCleanupSelector { + /// The maximal number of entries that this will remove. + /// + /// Needed for weight calculation. + pub fn limit(&self) -> u32 { + match self { + Self::Specific(ids) => ids.len() as u32, + Self::Wildcard { limit, .. } => limit.unwrap_or(DEFAULT_HISTORIC_BATCH_CLEAR_SIZE), + } + } +} + +/// Convenience alias for [`MigrationCursor`]. +pub type CursorOf = MigrationCursor, BlockNumberFor>; + +/// Convenience alias for the raw inner cursor of a migration. +pub type RawCursorOf = BoundedVec::CursorMaxLen>; + +/// Convenience alias for the identifier of a migration. +pub type IdentifierOf = BoundedVec::IdentifierMaxLen>; + +/// Convenience alias for [`ActiveCursor`]. +pub type ActiveCursorOf = ActiveCursor, BlockNumberFor>; + +/// Trait for a tuple of No-OP migrations with one element. +pub trait MockedMigrations: SteppedMigrations { + /// The migration should fail after `n` steps. + fn set_fail_after(n: u32); + /// The migration should succeed after `n` steps. + fn set_success_after(n: u32); +} + +#[frame_support::pallet] +pub mod pallet { + use super::*; + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; + + #[pallet::pallet] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: frame_system::Config { + /// The overarching event type of the runtime. + type RuntimeEvent: From> + IsType<::RuntimeEvent>; + + /// All the multi-block migrations to run. + /// + /// Should only be updated in a runtime-upgrade once all the old migrations have completed. + /// (Check that [`Cursor`] is `None`). + #[cfg(not(feature = "runtime-benchmarks"))] + type Migrations: SteppedMigrations; + + /// Mocked migrations for benchmarking only. + /// + /// Should be configured to [`crate::mock_helpers::MockedMigrations`] in benchmarks. + #[cfg(feature = "runtime-benchmarks")] + type Migrations: MockedMigrations; + + /// The maximal length of an encoded cursor. + /// + /// A good default needs to selected such that no migration will ever have a cursor with MEL + /// above this limit. This is statically checked in `integrity_test`. + #[pallet::constant] + type CursorMaxLen: Get; + + /// The maximal length of an encoded identifier. + /// + /// A good default needs to selected such that no migration will ever have an identifier + /// with MEL above this limit. This is statically checked in `integrity_test`. + #[pallet::constant] + type IdentifierMaxLen: Get; + + /// Notifications for status updates of a runtime upgrade. + /// + /// Could be used to pause XCM etc. + type MigrationStatusHandler: MigrationStatusHandler; + + /// Handler for failed migrations. + type FailedMigrationHandler: FailedMigrationHandler; + + /// The maximum weight to spend each block to execute migrations. + type MaxServiceWeight: Get; + + /// Weight information for the calls and functions of this pallet. + type WeightInfo: WeightInfo; + } + + /// The currently active migration to run and its cursor. + /// + /// `None` indicates that no migration is running. + #[pallet::storage] + pub type Cursor = StorageValue<_, CursorOf, OptionQuery>; + + /// Set of all successfully executed migrations. + /// + /// This is used as blacklist, to not re-execute migrations that have not been removed from the + /// codebase yet. Governance can regularly clear this out via `clear_historic`. + #[pallet::storage] + pub type Historic = StorageMap<_, Twox64Concat, IdentifierOf, (), OptionQuery>; + + #[pallet::event] + #[pallet::generate_deposit(pub(super) fn deposit_event)] + pub enum Event { + /// A Runtime upgrade started. + /// + /// Its end is indicated by `UpgradeCompleted` or `UpgradeFailed`. + UpgradeStarted { + /// The number of migrations that this upgrade contains. + /// + /// This can be used to design a progress indicator in combination with counting the + /// `MigrationCompleted` and `MigrationSkipped` events. + migrations: u32, + }, + /// The current runtime upgrade completed. + /// + /// This implies that all of its migrations completed successfully as well. + UpgradeCompleted, + /// Runtime upgrade failed. + /// + /// This is very bad and will require governance intervention. + UpgradeFailed, + /// A migration was skipped since it was already executed in the past. + MigrationSkipped { + /// The index of the skipped migration within the [`Config::Migrations`] list. + index: u32, + }, + /// A migration progressed. + MigrationAdvanced { + /// The index of the migration within the [`Config::Migrations`] list. + index: u32, + /// The number of blocks that this migration took so far. + took: BlockNumberFor, + }, + /// A Migration completed. + MigrationCompleted { + /// The index of the migration within the [`Config::Migrations`] list. + index: u32, + /// The number of blocks that this migration took so far. + took: BlockNumberFor, + }, + /// A Migration failed. + /// + /// This implies that the whole upgrade failed and governance intervention is required. + MigrationFailed { + /// The index of the migration within the [`Config::Migrations`] list. + index: u32, + /// The number of blocks that this migration took so far. + took: BlockNumberFor, + }, + /// The set of historical migrations has been cleared. + HistoricCleared { + /// Should be passed to `clear_historic` in a successive call. + next_cursor: Option>, + }, + } + + #[pallet::error] + pub enum Error { + /// The operation cannot complete since some MBMs are ongoing. + Ongoing, + } + + #[pallet::hooks] + impl Hooks> for Pallet { + fn on_runtime_upgrade() -> Weight { + Self::onboard_new_mbms() + } + + #[cfg(feature = "std")] + fn integrity_test() { + // Check that the migrations tuple is legit. + frame_support::assert_ok!(T::Migrations::integrity_test()); + + // Very important! Ensure that the pallet is configured in `System::Config`. + { + assert!(!Cursor::::exists(), "Externalities storage should be clean"); + assert!(!::MultiBlockMigrator::ongoing()); + + Cursor::::put(MigrationCursor::Stuck); + assert!(::MultiBlockMigrator::ongoing()); + + Cursor::::kill(); + } + + // The per-block service weight is sane. + #[cfg(not(test))] + { + let want = T::MaxServiceWeight::get(); + let max = ::BlockWeights::get().max_block; + + assert!(want.all_lte(max), "Service weight is larger than a block: {want} > {max}",); + } + + // Cursor MEL + { + let mel = T::Migrations::cursor_max_encoded_len(); + let max_mel = T::CursorMaxLen::get() as usize; + assert!( + mel <= max_mel, + "A Cursor is not guaranteed to fit into the storage: {mel} > {max_mel}", + ); + } + + // Identifier MEL + { + let mel = T::Migrations::identifier_max_encoded_len(); + let max_mel = T::IdentifierMaxLen::get() as usize; + assert!( + mel <= max_mel, + "An Identifier is not guaranteed to fit into the storage: {mel} > {max_mel}", + ); + } + } + } + + #[pallet::call(weight = T::WeightInfo)] + impl Pallet { + /// Allows root to set a cursor to forcefully start, stop or forward the migration process. + /// + /// Should normally not be needed and is only in place as emergency measure. Note that + /// restarting the migration process in this manner will not call the + /// [`MigrationStatusHandler::started`] hook or emit an `UpgradeStarted` event. + #[pallet::call_index(0)] + pub fn force_set_cursor( + origin: OriginFor, + cursor: Option>, + ) -> DispatchResult { + ensure_root(origin)?; + + Cursor::::set(cursor); + + Ok(()) + } + + /// Allows root to set an active cursor to forcefully start/forward the migration process. + /// + /// This is an edge-case version of [`Self::force_set_cursor`] that allows to set the + /// `started_at` value to the next block number. Otherwise this would not be possible, since + /// `force_set_cursor` takes an absolute block number. Setting `started_at` to `None` + /// indicates that the current block number plus one should be used. + #[pallet::call_index(1)] + pub fn force_set_active_cursor( + origin: OriginFor, + index: u32, + inner_cursor: Option>, + started_at: Option>, + ) -> DispatchResult { + ensure_root(origin)?; + + let started_at = started_at.unwrap_or( + System::::block_number().saturating_add(sp_runtime::traits::One::one()), + ); + Cursor::::put(MigrationCursor::Active(ActiveCursor { + index, + inner_cursor, + started_at, + })); + + Ok(()) + } + + /// Forces the onboarding of the migrations. + /// + /// This process happens automatically on a runtime upgrade. It is in place as an emergency + /// measurement. The cursor needs to be `None` for this to succeed. + #[pallet::call_index(2)] + pub fn force_onboard_mbms(origin: OriginFor) -> DispatchResult { + ensure_root(origin)?; + + ensure!(!Cursor::::exists(), Error::::Ongoing); + Self::onboard_new_mbms(); + + Ok(()) + } + + /// Clears the `Historic` set. + /// + /// `map_cursor` must be set to the last value that was returned by the + /// `HistoricCleared` event. The first time `None` can be used. `limit` must be chosen in a + /// way that will result in a sensible weight. + #[pallet::call_index(3)] + #[pallet::weight(T::WeightInfo::clear_historic(selector.limit()))] + pub fn clear_historic( + origin: OriginFor, + selector: HistoricCleanupSelector>, + ) -> DispatchResult { + ensure_root(origin)?; + + match &selector { + HistoricCleanupSelector::Specific(ids) => { + for id in ids { + Historic::::remove(id); + } + Self::deposit_event(Event::HistoricCleared { next_cursor: None }); + }, + HistoricCleanupSelector::Wildcard { previous_cursor, .. } => { + let next = Historic::::clear(selector.limit(), previous_cursor.as_deref()); + Self::deposit_event(Event::HistoricCleared { next_cursor: next.maybe_cursor }); + }, + } + + Ok(()) + } + } +} + +impl Pallet { + /// Onboard all new Multi-Block-Migrations and start the process of executing them. + /// + /// Should only be called once all previous migrations completed. + fn onboard_new_mbms() -> Weight { + if let Some(cursor) = Cursor::::get() { + log::error!("Ongoing migrations interrupted - chain stuck"); + + let maybe_index = cursor.as_active().map(|c| c.index); + Self::upgrade_failed(maybe_index); + return T::WeightInfo::onboard_new_mbms() + } + + let migrations = T::Migrations::len(); + log::debug!("Onboarding {migrations} new MBM migrations"); + + if migrations > 0 { + // Set the cursor to the first migration: + Cursor::::set(Some( + ActiveCursor { + index: 0, + inner_cursor: None, + started_at: System::::block_number(), + } + .into(), + )); + Self::deposit_event(Event::UpgradeStarted { migrations }); + T::MigrationStatusHandler::started(); + } + + T::WeightInfo::onboard_new_mbms() + } + + /// Tries to make progress on the Multi-Block-Migrations process. + fn progress_mbms(n: BlockNumberFor) -> Weight { + let mut meter = WeightMeter::with_limit(T::MaxServiceWeight::get()); + meter.consume(T::WeightInfo::progress_mbms_none()); + + let mut cursor = match Cursor::::get() { + None => { + log::trace!("[Block {n:?}] Waiting for cursor to become `Some`."); + return meter.consumed() + }, + Some(MigrationCursor::Active(cursor)) => { + log::debug!("Progressing MBM #{}", cursor.index); + cursor + }, + Some(MigrationCursor::Stuck) => { + log::error!("Migration stuck. Governance intervention required."); + return meter.consumed() + }, + }; + debug_assert!(Self::ongoing()); + + // The limit here is a defensive measure to prevent an infinite loop. It expresses that we + // allow no more than 8 MBMs to finish in a single block. This should be harmless, since we + // generally expect *Multi*-Block-Migrations to take *multiple* blocks. + for i in 0..8 { + match Self::exec_migration(cursor, i == 0, &mut meter) { + None => return meter.consumed(), + Some(ControlFlow::Continue(next_cursor)) => { + cursor = next_cursor; + }, + Some(ControlFlow::Break(last_cursor)) => { + cursor = last_cursor; + break + }, + } + } + + Cursor::::set(Some(cursor.into())); + + meter.consumed() + } + + /// Try to make progress on the current migration. + /// + /// Returns whether processing should continue or break for this block. The return value means: + /// - `None`: The migration process is completely finished. + /// - `ControlFlow::Break`: Continue in the *next* block with the given cursor. + /// - `ControlFlow::Continue`: Continue in the *current* block with the given cursor. + fn exec_migration( + mut cursor: ActiveCursorOf, + is_first: bool, + meter: &mut WeightMeter, + ) -> Option, ActiveCursorOf>> { + // The differences between the single branches' weights is not that big. And since we do + // only one step per block, we can just use the maximum instead of more precise accounting. + if meter.try_consume(Self::exec_migration_max_weight()).is_err() { + defensive_assert!(!is_first, "There should be enough weight to do this at least once"); + return Some(ControlFlow::Break(cursor)) + } + + let Some(id) = T::Migrations::nth_id(cursor.index) else { + // No more migrations in the tuple - we are done. + defensive_assert!(cursor.index == T::Migrations::len(), "Inconsistent MBMs tuple"); + Self::deposit_event(Event::UpgradeCompleted); + Cursor::::kill(); + T::MigrationStatusHandler::completed(); + return None; + }; + + let Ok(bounded_id): Result, _> = id.try_into() else { + defensive!("integrity_test ensures that all identifiers' MEL bounds fit into CursorMaxLen; qed."); + Self::upgrade_failed(Some(cursor.index)); + return None + }; + + if Historic::::contains_key(&bounded_id) { + Self::deposit_event(Event::MigrationSkipped { index: cursor.index }); + cursor.goto_next_migration(System::::block_number()); + return Some(ControlFlow::Continue(cursor)) + } + + let max_steps = T::Migrations::nth_max_steps(cursor.index); + let next_cursor = T::Migrations::nth_transactional_step( + cursor.index, + cursor.inner_cursor.clone().map(|c| c.into_inner()), + meter, + ); + let Some((max_steps, next_cursor)) = max_steps.zip(next_cursor) else { + defensive!("integrity_test ensures that the tuple is valid; qed"); + Self::upgrade_failed(Some(cursor.index)); + return None + }; + + let took = System::::block_number().saturating_sub(cursor.started_at); + match next_cursor { + Ok(Some(next_cursor)) => { + let Ok(bound_next_cursor) = next_cursor.try_into() else { + defensive!("The integrity check ensures that all cursors' MEL bound fits into CursorMaxLen; qed"); + Self::upgrade_failed(Some(cursor.index)); + return None + }; + + Self::deposit_event(Event::MigrationAdvanced { index: cursor.index, took }); + cursor.inner_cursor = Some(bound_next_cursor); + + if max_steps.map_or(false, |max| took > max.into()) { + Self::deposit_event(Event::MigrationFailed { index: cursor.index, took }); + Self::upgrade_failed(Some(cursor.index)); + None + } else { + // A migration cannot progress more than one step per block, we therefore break. + Some(ControlFlow::Break(cursor)) + } + }, + Ok(None) => { + // A migration is done when it returns cursor `None`. + Self::deposit_event(Event::MigrationCompleted { index: cursor.index, took }); + Historic::::insert(&bounded_id, ()); + cursor.goto_next_migration(System::::block_number()); + Some(ControlFlow::Continue(cursor)) + }, + Err(SteppedMigrationError::InsufficientWeight { required }) => { + if is_first || required.any_gt(meter.limit()) { + Self::deposit_event(Event::MigrationFailed { index: cursor.index, took }); + Self::upgrade_failed(Some(cursor.index)); + None + } else { + // Retry and hope that there is more weight in the next block. + Some(ControlFlow::Break(cursor)) + } + }, + Err(SteppedMigrationError::InvalidCursor | SteppedMigrationError::Failed) => { + Self::deposit_event(Event::MigrationFailed { index: cursor.index, took }); + Self::upgrade_failed(Some(cursor.index)); + None + }, + } + } + + /// Fail the current runtime upgrade, caused by `migration`. + fn upgrade_failed(migration: Option) { + use FailedMigrationHandling::*; + Self::deposit_event(Event::UpgradeFailed); + + match T::FailedMigrationHandler::failed(migration) { + KeepStuck => Cursor::::set(Some(MigrationCursor::Stuck)), + ForceUnstuck => Cursor::::kill(), + Ignore => {}, + } + } + + fn exec_migration_max_weight() -> Weight { + T::WeightInfo::exec_migration_complete() + .max(T::WeightInfo::exec_migration_completed()) + .max(T::WeightInfo::exec_migration_skipped_historic()) + .max(T::WeightInfo::exec_migration_advance()) + .max(T::WeightInfo::exec_migration_fail()) + } +} + +impl MultiStepMigrator for Pallet { + fn ongoing() -> bool { + Cursor::::exists() + } + + fn step() -> Weight { + Self::progress_mbms(System::::block_number()) + } +} diff --git a/substrate/frame/migrations/src/mock.rs b/substrate/frame/migrations/src/mock.rs new file mode 100644 index 000000000000..372926475544 --- /dev/null +++ b/substrate/frame/migrations/src/mock.rs @@ -0,0 +1,163 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Mocked runtime for testing the migrations pallet. + +#![cfg(test)] + +use crate::{mock_helpers::*, Event, Historic}; + +use frame_support::{ + derive_impl, + migrations::*, + traits::{OnFinalize, OnInitialize}, + weights::Weight, +}; +use frame_system::EventRecord; +use sp_core::{ConstU32, H256}; + +type Block = frame_system::mocking::MockBlock; + +// Configure a mock runtime to test the pallet. +frame_support::construct_runtime!( + pub enum Test { + System: frame_system, + Migrations: crate, + } +); + +#[derive_impl(frame_system::config_preludes::TestDefaultConfig as frame_system::DefaultConfig)] +impl frame_system::Config for Test { + type Block = Block; + type PalletInfo = PalletInfo; + type MultiBlockMigrator = Migrations; +} + +frame_support::parameter_types! { + pub const MaxServiceWeight: Weight = Weight::MAX.div(10); +} + +impl crate::Config for Test { + type RuntimeEvent = RuntimeEvent; + type Migrations = MockedMigrations; + type CursorMaxLen = ConstU32<65_536>; + type IdentifierMaxLen = ConstU32<256>; + type MigrationStatusHandler = MockedMigrationStatusHandler; + type FailedMigrationHandler = MockedFailedMigrationHandler; + type MaxServiceWeight = MaxServiceWeight; + type WeightInfo = (); +} + +frame_support::parameter_types! { + /// The number of started upgrades. + pub static UpgradesStarted: u32 = 0; + /// The number of completed upgrades. + pub static UpgradesCompleted: u32 = 0; + /// The migrations that failed. + pub static UpgradesFailed: Vec> = vec![]; + /// Return value of [`MockedFailedMigrationHandler::failed`]. + pub static FailedUpgradeResponse: FailedMigrationHandling = FailedMigrationHandling::KeepStuck; +} + +/// Records all started and completed upgrades in `UpgradesStarted` and `UpgradesCompleted`. +pub struct MockedMigrationStatusHandler; +impl MigrationStatusHandler for MockedMigrationStatusHandler { + fn started() { + log::info!("MigrationStatusHandler started"); + UpgradesStarted::mutate(|v| *v += 1); + } + + fn completed() { + log::info!("MigrationStatusHandler completed"); + UpgradesCompleted::mutate(|v| *v += 1); + } +} + +/// Records all failed upgrades in `UpgradesFailed`. +pub struct MockedFailedMigrationHandler; +impl FailedMigrationHandler for MockedFailedMigrationHandler { + fn failed(migration: Option) -> FailedMigrationHandling { + UpgradesFailed::mutate(|v| v.push(migration)); + let res = FailedUpgradeResponse::get(); + log::error!("FailedMigrationHandler failed at: {migration:?}, handling as {res:?}"); + res + } +} + +/// Returns the number of `(started, completed, failed)` upgrades and resets their numbers. +pub fn upgrades_started_completed_failed() -> (u32, u32, u32) { + (UpgradesStarted::take(), UpgradesCompleted::take(), UpgradesFailed::take().len() as u32) +} + +/// Build genesis storage according to the mock runtime. +pub fn new_test_ext() -> sp_io::TestExternalities { + sp_io::TestExternalities::new(Default::default()) +} + +/// Run this closure in test externalities. +pub fn test_closure(f: impl FnOnce() -> R) -> R { + let mut ext = new_test_ext(); + ext.execute_with(f) +} + +pub fn run_to_block(n: u32) { + while System::block_number() < n as u64 { + log::debug!("Block {}", System::block_number()); + System::set_block_number(System::block_number() + 1); + System::on_initialize(System::block_number()); + Migrations::on_initialize(System::block_number()); + // Executive calls this: + ::step(); + + Migrations::on_finalize(System::block_number()); + System::on_finalize(System::block_number()); + } +} + +/// Returns the historic migrations, sorted by their identifier. +pub fn historic() -> Vec { + let mut historic = Historic::::iter_keys().collect::>(); + historic.sort(); + historic +} + +// Traits to make using events less insufferable: +pub trait IntoRecord { + fn into_record(self) -> EventRecord<::RuntimeEvent, H256>; +} + +impl IntoRecord for Event { + fn into_record(self) -> EventRecord<::RuntimeEvent, H256> { + let re: ::RuntimeEvent = self.into(); + EventRecord { phase: frame_system::Phase::Initialization, event: re, topics: vec![] } + } +} + +pub trait IntoRecords { + fn into_records(self) -> Vec::RuntimeEvent, H256>>; +} + +impl IntoRecords for Vec { + fn into_records(self) -> Vec::RuntimeEvent, H256>> { + self.into_iter().map(|e| e.into_record()).collect() + } +} + +pub fn assert_events(events: Vec) { + pretty_assertions::assert_eq!(events.into_records(), System::events()); + System::reset_events(); +} diff --git a/substrate/frame/migrations/src/mock_helpers.rs b/substrate/frame/migrations/src/mock_helpers.rs new file mode 100644 index 000000000000..c5e23efb4e31 --- /dev/null +++ b/substrate/frame/migrations/src/mock_helpers.rs @@ -0,0 +1,142 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Test helpers for internal and external usage. + +#![allow(missing_docs)] + +use codec::{Decode, Encode}; +use frame_support::{ + migrations::*, + weights::{Weight, WeightMeter}, +}; +use sp_core::ConstU32; +use sp_runtime::BoundedVec; +use sp_std::{vec, vec::Vec}; + +/// Opaque identifier of a migration. +pub type MockedIdentifier = BoundedVec>; + +/// How a mocked migration should behave. +#[derive(Debug, Clone, Copy, Encode, Decode)] +pub enum MockedMigrationKind { + /// Succeed after its number of steps elapsed. + SucceedAfter, + /// Fail after its number of steps elapsed. + FailAfter, + /// Never terminate. + TimeoutAfter, + /// Cause an [`SteppedMigrationError::InsufficientWeight`] error after its number of steps + /// elapsed. + HighWeightAfter(Weight), +} +use MockedMigrationKind::*; // C style + +/// Creates a migration identifier with a specific `kind` and `steps`. +pub fn mocked_id(kind: MockedMigrationKind, steps: u32) -> MockedIdentifier { + (b"MockedMigration", kind, steps).encode().try_into().unwrap() +} + +frame_support::parameter_types! { + /// The configs for the migrations to run. + storage MIGRATIONS: Vec<(MockedMigrationKind, u32)> = vec![]; +} + +/// Allows to set the migrations to run at runtime instead of compile-time. +/// +/// It achieves this by using the storage to store the migrations to run. +pub struct MockedMigrations; +impl SteppedMigrations for MockedMigrations { + fn len() -> u32 { + MIGRATIONS::get().len() as u32 + } + + fn nth_id(n: u32) -> Option> { + let k = MIGRATIONS::get().get(n as usize).copied(); + k.map(|(kind, steps)| mocked_id(kind, steps).into_inner()) + } + + fn nth_step( + n: u32, + cursor: Option>, + _meter: &mut WeightMeter, + ) -> Option>, SteppedMigrationError>> { + let (kind, steps) = MIGRATIONS::get()[n as usize]; + + let mut count: u32 = + cursor.as_ref().and_then(|c| Decode::decode(&mut &c[..]).ok()).unwrap_or(0); + log::debug!("MockedMigration: Step {}", count); + if count != steps || matches!(kind, TimeoutAfter) { + count += 1; + return Some(Ok(Some(count.encode()))) + } + + Some(match kind { + SucceedAfter => { + log::debug!("MockedMigration: Succeeded after {} steps", count); + Ok(None) + }, + HighWeightAfter(required) => { + log::debug!("MockedMigration: Not enough weight after {} steps", count); + Err(SteppedMigrationError::InsufficientWeight { required }) + }, + FailAfter => { + log::debug!("MockedMigration: Failed after {} steps", count); + Err(SteppedMigrationError::Failed) + }, + TimeoutAfter => unreachable!(), + }) + } + + fn nth_transactional_step( + n: u32, + cursor: Option>, + meter: &mut WeightMeter, + ) -> Option>, SteppedMigrationError>> { + // This is a hack but should be fine. We dont need it in testing. + Self::nth_step(n, cursor, meter) + } + + fn nth_max_steps(n: u32) -> Option> { + MIGRATIONS::get().get(n as usize).map(|(_, s)| Some(*s)) + } + + fn cursor_max_encoded_len() -> usize { + 65_536 + } + + fn identifier_max_encoded_len() -> usize { + 256 + } +} + +impl MockedMigrations { + /// Set the migrations to run. + pub fn set(migrations: Vec<(MockedMigrationKind, u32)>) { + MIGRATIONS::set(&migrations); + } +} + +impl crate::MockedMigrations for MockedMigrations { + fn set_fail_after(steps: u32) { + MIGRATIONS::set(&vec![(FailAfter, steps)]); + } + + fn set_success_after(steps: u32) { + MIGRATIONS::set(&vec![(SucceedAfter, steps)]); + } +} diff --git a/substrate/frame/migrations/src/tests.rs b/substrate/frame/migrations/src/tests.rs new file mode 100644 index 000000000000..9c9043d37a62 --- /dev/null +++ b/substrate/frame/migrations/src/tests.rs @@ -0,0 +1,335 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![cfg(test)] + +use crate::{ + mock::{Test as T, *}, + mock_helpers::{MockedMigrationKind::*, *}, + Cursor, Event, FailedMigrationHandling, MigrationCursor, +}; +use frame_support::{pallet_prelude::Weight, traits::OnRuntimeUpgrade}; + +#[docify::export] +#[test] +fn simple_works() { + use Event::*; + test_closure(|| { + // Add three migrations, each taking one block longer than the previous. + MockedMigrations::set(vec![(SucceedAfter, 0), (SucceedAfter, 1), (SucceedAfter, 2)]); + + System::set_block_number(1); + Migrations::on_runtime_upgrade(); + run_to_block(10); + + // Check that the executed migrations are recorded in `Historical`. + assert_eq!( + historic(), + vec![ + mocked_id(SucceedAfter, 0), + mocked_id(SucceedAfter, 1), + mocked_id(SucceedAfter, 2), + ] + ); + + // Check that we got all events. + assert_events(vec![ + UpgradeStarted { migrations: 3 }, + MigrationCompleted { index: 0, took: 1 }, + MigrationAdvanced { index: 1, took: 0 }, + MigrationCompleted { index: 1, took: 1 }, + MigrationAdvanced { index: 2, took: 0 }, + MigrationAdvanced { index: 2, took: 1 }, + MigrationCompleted { index: 2, took: 2 }, + UpgradeCompleted, + ]); + }); +} + +#[test] +fn failing_migration_sets_cursor_to_stuck() { + test_closure(|| { + FailedUpgradeResponse::set(FailedMigrationHandling::KeepStuck); + MockedMigrations::set(vec![(FailAfter, 2)]); + + System::set_block_number(1); + Migrations::on_runtime_upgrade(); + run_to_block(10); + + // Failed migrations are not recorded in `Historical`. + assert!(historic().is_empty()); + // Check that we got all events. + assert_events(vec![ + Event::UpgradeStarted { migrations: 1 }, + Event::MigrationAdvanced { index: 0, took: 1 }, + Event::MigrationAdvanced { index: 0, took: 2 }, + Event::MigrationFailed { index: 0, took: 3 }, + Event::UpgradeFailed, + ]); + + // Check that the handler was called correctly. + assert_eq!(UpgradesStarted::take(), 1); + assert_eq!(UpgradesCompleted::take(), 0); + assert_eq!(UpgradesFailed::take(), vec![Some(0)]); + + assert_eq!(Cursor::::get(), Some(MigrationCursor::Stuck), "Must stuck the chain"); + }); +} + +#[test] +fn failing_migration_force_unstuck_works() { + test_closure(|| { + FailedUpgradeResponse::set(FailedMigrationHandling::ForceUnstuck); + MockedMigrations::set(vec![(FailAfter, 2)]); + + System::set_block_number(1); + Migrations::on_runtime_upgrade(); + run_to_block(10); + + // Failed migrations are not recorded in `Historical`. + assert!(historic().is_empty()); + // Check that we got all events. + assert_events(vec![ + Event::UpgradeStarted { migrations: 1 }, + Event::MigrationAdvanced { index: 0, took: 1 }, + Event::MigrationAdvanced { index: 0, took: 2 }, + Event::MigrationFailed { index: 0, took: 3 }, + Event::UpgradeFailed, + ]); + + // Check that the handler was called correctly. + assert_eq!(UpgradesStarted::take(), 1); + assert_eq!(UpgradesCompleted::take(), 0); + assert_eq!(UpgradesFailed::take(), vec![Some(0)]); + + assert!(Cursor::::get().is_none(), "Must unstuck the chain"); + }); +} + +/// A migration that reports not getting enough weight errors if it is the first one to run in that +/// block. +#[test] +fn high_weight_migration_singular_fails() { + test_closure(|| { + MockedMigrations::set(vec![(HighWeightAfter(Weight::zero()), 2)]); + + System::set_block_number(1); + Migrations::on_runtime_upgrade(); + run_to_block(10); + + // Failed migrations are not recorded in `Historical`. + assert!(historic().is_empty()); + // Check that we got all events. + assert_events(vec![ + Event::UpgradeStarted { migrations: 1 }, + Event::MigrationAdvanced { index: 0, took: 1 }, + Event::MigrationAdvanced { index: 0, took: 2 }, + Event::MigrationFailed { index: 0, took: 3 }, + Event::UpgradeFailed, + ]); + + // Check that the handler was called correctly. + assert_eq!(upgrades_started_completed_failed(), (1, 0, 1)); + assert_eq!(Cursor::::get(), Some(MigrationCursor::Stuck)); + }); +} + +/// A migration that reports of not getting enough weight is retried once, if it is not the first +/// one to run in a block. +#[test] +fn high_weight_migration_retries_once() { + test_closure(|| { + MockedMigrations::set(vec![(SucceedAfter, 0), (HighWeightAfter(Weight::zero()), 0)]); + + System::set_block_number(1); + Migrations::on_runtime_upgrade(); + run_to_block(10); + + assert_eq!(historic(), vec![mocked_id(SucceedAfter, 0)]); + // Check that we got all events. + assert_events::>(vec![ + Event::UpgradeStarted { migrations: 2 }, + Event::MigrationCompleted { index: 0, took: 1 }, + // `took=1` means that it was retried once. + Event::MigrationFailed { index: 1, took: 1 }, + Event::UpgradeFailed, + ]); + + // Check that the handler was called correctly. + assert_eq!(upgrades_started_completed_failed(), (1, 0, 1)); + assert_eq!(Cursor::::get(), Some(MigrationCursor::Stuck)); + }); +} + +/// If a migration uses more weight than the limit, then it will not retry but fail even when it is +/// not the first one in the block. +// Note: Same as `high_weight_migration_retries_once` but with different required weight for the +// migration. +#[test] +fn high_weight_migration_permanently_overweight_fails() { + test_closure(|| { + MockedMigrations::set(vec![(SucceedAfter, 0), (HighWeightAfter(Weight::MAX), 0)]); + + System::set_block_number(1); + Migrations::on_runtime_upgrade(); + run_to_block(10); + + assert_eq!(historic(), vec![mocked_id(SucceedAfter, 0)]); + // Check that we got all events. + assert_events::>(vec![ + Event::UpgradeStarted { migrations: 2 }, + Event::MigrationCompleted { index: 0, took: 1 }, + // `blocks=0` means that it was not retried. + Event::MigrationFailed { index: 1, took: 0 }, + Event::UpgradeFailed, + ]); + + // Check that the handler was called correctly. + assert_eq!(upgrades_started_completed_failed(), (1, 0, 1)); + assert_eq!(Cursor::::get(), Some(MigrationCursor::Stuck)); + }); +} + +#[test] +fn historic_skipping_works() { + test_closure(|| { + MockedMigrations::set(vec![ + (SucceedAfter, 0), + (SucceedAfter, 0), // duplicate + (SucceedAfter, 1), + (SucceedAfter, 2), + (SucceedAfter, 1), // duplicate + ]); + + System::set_block_number(1); + Migrations::on_runtime_upgrade(); + run_to_block(10); + + // Just three historical ones, since two were added twice. + assert_eq!( + historic(), + vec![ + mocked_id(SucceedAfter, 0), + mocked_id(SucceedAfter, 1), + mocked_id(SucceedAfter, 2), + ] + ); + // Events received. + assert_events(vec![ + Event::UpgradeStarted { migrations: 5 }, + Event::MigrationCompleted { index: 0, took: 1 }, + Event::MigrationSkipped { index: 1 }, + Event::MigrationAdvanced { index: 2, took: 0 }, + Event::MigrationCompleted { index: 2, took: 1 }, + Event::MigrationAdvanced { index: 3, took: 0 }, + Event::MigrationAdvanced { index: 3, took: 1 }, + Event::MigrationCompleted { index: 3, took: 2 }, + Event::MigrationSkipped { index: 4 }, + Event::UpgradeCompleted, + ]); + assert_eq!(upgrades_started_completed_failed(), (1, 1, 0)); + + // Now go for another upgrade; just to make sure that it wont execute again. + System::reset_events(); + Migrations::on_runtime_upgrade(); + run_to_block(20); + + // Same historical ones as before. + assert_eq!( + historic(), + vec![ + mocked_id(SucceedAfter, 0), + mocked_id(SucceedAfter, 1), + mocked_id(SucceedAfter, 2), + ] + ); + + // Everything got skipped. + assert_events(vec![ + Event::UpgradeStarted { migrations: 5 }, + Event::MigrationSkipped { index: 0 }, + Event::MigrationSkipped { index: 1 }, + Event::MigrationSkipped { index: 2 }, + Event::MigrationSkipped { index: 3 }, + Event::MigrationSkipped { index: 4 }, + Event::UpgradeCompleted, + ]); + assert_eq!(upgrades_started_completed_failed(), (1, 1, 0)); + }); +} + +/// When another upgrade happens while a migration is still running, it should set the cursor to +/// stuck. +#[test] +fn upgrade_fails_when_migration_active() { + test_closure(|| { + MockedMigrations::set(vec![(SucceedAfter, 10)]); + + System::set_block_number(1); + Migrations::on_runtime_upgrade(); + run_to_block(3); + + // Events received. + assert_events(vec![ + Event::UpgradeStarted { migrations: 1 }, + Event::MigrationAdvanced { index: 0, took: 1 }, + Event::MigrationAdvanced { index: 0, took: 2 }, + ]); + assert_eq!(upgrades_started_completed_failed(), (1, 0, 0)); + + // Upgrade again. + Migrations::on_runtime_upgrade(); + // -- Defensive path -- + assert_eq!(Cursor::::get(), Some(MigrationCursor::Stuck)); + assert_events(vec![Event::UpgradeFailed]); + assert_eq!(upgrades_started_completed_failed(), (0, 0, 1)); + }); +} + +#[test] +fn migration_timeout_errors() { + test_closure(|| { + MockedMigrations::set(vec![(TimeoutAfter, 3)]); + + System::set_block_number(1); + Migrations::on_runtime_upgrade(); + run_to_block(5); + + // Times out after taking more than 3 steps. + assert_events(vec![ + Event::UpgradeStarted { migrations: 1 }, + Event::MigrationAdvanced { index: 0, took: 1 }, + Event::MigrationAdvanced { index: 0, took: 2 }, + Event::MigrationAdvanced { index: 0, took: 3 }, + Event::MigrationAdvanced { index: 0, took: 4 }, + Event::MigrationFailed { index: 0, took: 4 }, + Event::UpgradeFailed, + ]); + assert_eq!(upgrades_started_completed_failed(), (1, 0, 1)); + + // Failed migrations are not black-listed. + assert!(historic().is_empty()); + assert_eq!(Cursor::::get(), Some(MigrationCursor::Stuck)); + + Migrations::on_runtime_upgrade(); + run_to_block(6); + + assert_events(vec![Event::UpgradeFailed]); + assert_eq!(Cursor::::get(), Some(MigrationCursor::Stuck)); + assert_eq!(upgrades_started_completed_failed(), (0, 0, 1)); + }); +} diff --git a/substrate/frame/migrations/src/weights.rs b/substrate/frame/migrations/src/weights.rs new file mode 100644 index 000000000000..c9b63258c44b --- /dev/null +++ b/substrate/frame/migrations/src/weights.rs @@ -0,0 +1,358 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Autogenerated weights for `pallet_migrations` +//! +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev +//! DATE: 2023-12-04, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! WORST CASE MAP SIZE: `1000000` +//! HOSTNAME: `loud1`, CPU: `AMD EPYC 7282 16-Core Processor` +//! WASM-EXECUTION: `Compiled`, CHAIN: `Some("dev")`, DB CACHE: `1024` + +// Executed Command: +// target/release/substrate-node +// benchmark +// pallet +// --chain +// dev +// --pallet +// pallet-migrations +// --extrinsic +// +// --output +// weight.rs +// --template +// ../../polkadot-sdk/substrate/.maintain/frame-weight-template.hbs + +#![cfg_attr(rustfmt, rustfmt_skip)] +#![allow(unused_parens)] +#![allow(unused_imports)] +#![allow(missing_docs)] + +use frame_support::{traits::Get, weights::{Weight, constants::RocksDbWeight}}; +use core::marker::PhantomData; + +/// Weight functions needed for `pallet_migrations`. +pub trait WeightInfo { + fn onboard_new_mbms() -> Weight; + fn progress_mbms_none() -> Weight; + fn exec_migration_completed() -> Weight; + fn exec_migration_skipped_historic() -> Weight; + fn exec_migration_advance() -> Weight; + fn exec_migration_complete() -> Weight; + fn exec_migration_fail() -> Weight; + fn on_init_loop() -> Weight; + fn force_set_cursor() -> Weight; + fn force_set_active_cursor() -> Weight; + fn force_onboard_mbms() -> Weight; + fn clear_historic(n: u32, ) -> Weight; +} + +/// Weights for `pallet_migrations` using the Substrate node and recommended hardware. +pub struct SubstrateWeight(PhantomData); +impl WeightInfo for SubstrateWeight { + /// Storage: `MultiBlockMigrations::Cursor` (r:1 w:1) + /// Proof: `MultiBlockMigrations::Cursor` (`max_values`: Some(1), `max_size`: Some(65550), added: 66045, mode: `MaxEncodedLen`) + /// Storage: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Proof: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + fn onboard_new_mbms() -> Weight { + // Proof Size summary in bytes: + // Measured: `243` + // Estimated: `67035` + // Minimum execution time: 13_980_000 picoseconds. + Weight::from_parts(14_290_000, 67035) + .saturating_add(T::DbWeight::get().reads(2_u64)) + .saturating_add(T::DbWeight::get().writes(1_u64)) + } + /// Storage: `MultiBlockMigrations::Cursor` (r:1 w:0) + /// Proof: `MultiBlockMigrations::Cursor` (`max_values`: Some(1), `max_size`: Some(65550), added: 66045, mode: `MaxEncodedLen`) + fn progress_mbms_none() -> Weight { + // Proof Size summary in bytes: + // Measured: `109` + // Estimated: `67035` + // Minimum execution time: 3_770_000 picoseconds. + Weight::from_parts(4_001_000, 67035) + .saturating_add(T::DbWeight::get().reads(1_u64)) + } + /// Storage: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Proof: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Storage: `MultiBlockMigrations::Cursor` (r:0 w:1) + /// Proof: `MultiBlockMigrations::Cursor` (`max_values`: Some(1), `max_size`: Some(65550), added: 66045, mode: `MaxEncodedLen`) + fn exec_migration_completed() -> Weight { + // Proof Size summary in bytes: + // Measured: `134` + // Estimated: `3599` + // Minimum execution time: 10_900_000 picoseconds. + Weight::from_parts(11_251_000, 3599) + .saturating_add(T::DbWeight::get().reads(1_u64)) + .saturating_add(T::DbWeight::get().writes(1_u64)) + } + /// Storage: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Proof: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Storage: `MultiBlockMigrations::Historic` (r:1 w:0) + /// Proof: `MultiBlockMigrations::Historic` (`max_values`: None, `max_size`: Some(266), added: 2741, mode: `MaxEncodedLen`) + fn exec_migration_skipped_historic() -> Weight { + // Proof Size summary in bytes: + // Measured: `297` + // Estimated: `3762` + // Minimum execution time: 17_891_000 picoseconds. + Weight::from_parts(18_501_000, 3762) + .saturating_add(T::DbWeight::get().reads(2_u64)) + } + /// Storage: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Proof: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Storage: `MultiBlockMigrations::Historic` (r:1 w:0) + /// Proof: `MultiBlockMigrations::Historic` (`max_values`: None, `max_size`: Some(266), added: 2741, mode: `MaxEncodedLen`) + fn exec_migration_advance() -> Weight { + // Proof Size summary in bytes: + // Measured: `243` + // Estimated: `3731` + // Minimum execution time: 18_271_000 picoseconds. + Weight::from_parts(18_740_000, 3731) + .saturating_add(T::DbWeight::get().reads(2_u64)) + } + /// Storage: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Proof: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Storage: `MultiBlockMigrations::Historic` (r:1 w:1) + /// Proof: `MultiBlockMigrations::Historic` (`max_values`: None, `max_size`: Some(266), added: 2741, mode: `MaxEncodedLen`) + fn exec_migration_complete() -> Weight { + // Proof Size summary in bytes: + // Measured: `243` + // Estimated: `3731` + // Minimum execution time: 21_241_000 picoseconds. + Weight::from_parts(21_911_000, 3731) + .saturating_add(T::DbWeight::get().reads(2_u64)) + .saturating_add(T::DbWeight::get().writes(1_u64)) + } + /// Storage: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Proof: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Storage: `MultiBlockMigrations::Historic` (r:1 w:0) + /// Proof: `MultiBlockMigrations::Historic` (`max_values`: None, `max_size`: Some(266), added: 2741, mode: `MaxEncodedLen`) + /// Storage: `MultiBlockMigrations::Cursor` (r:0 w:1) + /// Proof: `MultiBlockMigrations::Cursor` (`max_values`: Some(1), `max_size`: Some(65550), added: 66045, mode: `MaxEncodedLen`) + fn exec_migration_fail() -> Weight { + // Proof Size summary in bytes: + // Measured: `243` + // Estimated: `3731` + // Minimum execution time: 22_740_000 picoseconds. + Weight::from_parts(23_231_000, 3731) + .saturating_add(T::DbWeight::get().reads(2_u64)) + .saturating_add(T::DbWeight::get().writes(1_u64)) + } + fn on_init_loop() -> Weight { + // Proof Size summary in bytes: + // Measured: `0` + // Estimated: `0` + // Minimum execution time: 440_000 picoseconds. + Weight::from_parts(500_000, 0) + } + /// Storage: `MultiBlockMigrations::Cursor` (r:0 w:1) + /// Proof: `MultiBlockMigrations::Cursor` (`max_values`: Some(1), `max_size`: Some(65550), added: 66045, mode: `MaxEncodedLen`) + fn force_set_cursor() -> Weight { + // Proof Size summary in bytes: + // Measured: `0` + // Estimated: `0` + // Minimum execution time: 5_751_000 picoseconds. + Weight::from_parts(5_950_000, 0) + .saturating_add(T::DbWeight::get().writes(1_u64)) + } + /// Storage: `MultiBlockMigrations::Cursor` (r:0 w:1) + /// Proof: `MultiBlockMigrations::Cursor` (`max_values`: Some(1), `max_size`: Some(65550), added: 66045, mode: `MaxEncodedLen`) + fn force_set_active_cursor() -> Weight { + // Proof Size summary in bytes: + // Measured: `0` + // Estimated: `0` + // Minimum execution time: 6_350_000 picoseconds. + Weight::from_parts(6_560_000, 0) + .saturating_add(T::DbWeight::get().writes(1_u64)) + } + /// Storage: `MultiBlockMigrations::Cursor` (r:1 w:0) + /// Proof: `MultiBlockMigrations::Cursor` (`max_values`: Some(1), `max_size`: Some(65550), added: 66045, mode: `MaxEncodedLen`) + /// Storage: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Proof: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + fn force_onboard_mbms() -> Weight { + // Proof Size summary in bytes: + // Measured: `218` + // Estimated: `67035` + // Minimum execution time: 11_121_000 picoseconds. + Weight::from_parts(11_530_000, 67035) + .saturating_add(T::DbWeight::get().reads(2_u64)) + } + /// Storage: `MultiBlockMigrations::Historic` (r:256 w:256) + /// Proof: `MultiBlockMigrations::Historic` (`max_values`: None, `max_size`: Some(266), added: 2741, mode: `MaxEncodedLen`) + /// The range of component `n` is `[0, 256]`. + fn clear_historic(n: u32, ) -> Weight { + // Proof Size summary in bytes: + // Measured: `1089 + n * (271 ±0)` + // Estimated: `3834 + n * (2740 ±0)` + // Minimum execution time: 21_891_000 picoseconds. + Weight::from_parts(18_572_306, 3834) + // Standard Error: 3_236 + .saturating_add(Weight::from_parts(1_648_429, 0).saturating_mul(n.into())) + .saturating_add(T::DbWeight::get().reads(1_u64)) + .saturating_add(T::DbWeight::get().reads((1_u64).saturating_mul(n.into()))) + .saturating_add(T::DbWeight::get().writes((1_u64).saturating_mul(n.into()))) + .saturating_add(Weight::from_parts(0, 2740).saturating_mul(n.into())) + } +} + +// For backwards compatibility and tests. +impl WeightInfo for () { + /// Storage: `MultiBlockMigrations::Cursor` (r:1 w:1) + /// Proof: `MultiBlockMigrations::Cursor` (`max_values`: Some(1), `max_size`: Some(65550), added: 66045, mode: `MaxEncodedLen`) + /// Storage: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Proof: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + fn onboard_new_mbms() -> Weight { + // Proof Size summary in bytes: + // Measured: `243` + // Estimated: `67035` + // Minimum execution time: 13_980_000 picoseconds. + Weight::from_parts(14_290_000, 67035) + .saturating_add(RocksDbWeight::get().reads(2_u64)) + .saturating_add(RocksDbWeight::get().writes(1_u64)) + } + /// Storage: `MultiBlockMigrations::Cursor` (r:1 w:0) + /// Proof: `MultiBlockMigrations::Cursor` (`max_values`: Some(1), `max_size`: Some(65550), added: 66045, mode: `MaxEncodedLen`) + fn progress_mbms_none() -> Weight { + // Proof Size summary in bytes: + // Measured: `109` + // Estimated: `67035` + // Minimum execution time: 3_770_000 picoseconds. + Weight::from_parts(4_001_000, 67035) + .saturating_add(RocksDbWeight::get().reads(1_u64)) + } + /// Storage: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Proof: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Storage: `MultiBlockMigrations::Cursor` (r:0 w:1) + /// Proof: `MultiBlockMigrations::Cursor` (`max_values`: Some(1), `max_size`: Some(65550), added: 66045, mode: `MaxEncodedLen`) + fn exec_migration_completed() -> Weight { + // Proof Size summary in bytes: + // Measured: `134` + // Estimated: `3599` + // Minimum execution time: 10_900_000 picoseconds. + Weight::from_parts(11_251_000, 3599) + .saturating_add(RocksDbWeight::get().reads(1_u64)) + .saturating_add(RocksDbWeight::get().writes(1_u64)) + } + /// Storage: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Proof: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Storage: `MultiBlockMigrations::Historic` (r:1 w:0) + /// Proof: `MultiBlockMigrations::Historic` (`max_values`: None, `max_size`: Some(266), added: 2741, mode: `MaxEncodedLen`) + fn exec_migration_skipped_historic() -> Weight { + // Proof Size summary in bytes: + // Measured: `297` + // Estimated: `3762` + // Minimum execution time: 17_891_000 picoseconds. + Weight::from_parts(18_501_000, 3762) + .saturating_add(RocksDbWeight::get().reads(2_u64)) + } + /// Storage: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Proof: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Storage: `MultiBlockMigrations::Historic` (r:1 w:0) + /// Proof: `MultiBlockMigrations::Historic` (`max_values`: None, `max_size`: Some(266), added: 2741, mode: `MaxEncodedLen`) + fn exec_migration_advance() -> Weight { + // Proof Size summary in bytes: + // Measured: `243` + // Estimated: `3731` + // Minimum execution time: 18_271_000 picoseconds. + Weight::from_parts(18_740_000, 3731) + .saturating_add(RocksDbWeight::get().reads(2_u64)) + } + /// Storage: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Proof: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Storage: `MultiBlockMigrations::Historic` (r:1 w:1) + /// Proof: `MultiBlockMigrations::Historic` (`max_values`: None, `max_size`: Some(266), added: 2741, mode: `MaxEncodedLen`) + fn exec_migration_complete() -> Weight { + // Proof Size summary in bytes: + // Measured: `243` + // Estimated: `3731` + // Minimum execution time: 21_241_000 picoseconds. + Weight::from_parts(21_911_000, 3731) + .saturating_add(RocksDbWeight::get().reads(2_u64)) + .saturating_add(RocksDbWeight::get().writes(1_u64)) + } + /// Storage: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Proof: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Storage: `MultiBlockMigrations::Historic` (r:1 w:0) + /// Proof: `MultiBlockMigrations::Historic` (`max_values`: None, `max_size`: Some(266), added: 2741, mode: `MaxEncodedLen`) + /// Storage: `MultiBlockMigrations::Cursor` (r:0 w:1) + /// Proof: `MultiBlockMigrations::Cursor` (`max_values`: Some(1), `max_size`: Some(65550), added: 66045, mode: `MaxEncodedLen`) + fn exec_migration_fail() -> Weight { + // Proof Size summary in bytes: + // Measured: `243` + // Estimated: `3731` + // Minimum execution time: 22_740_000 picoseconds. + Weight::from_parts(23_231_000, 3731) + .saturating_add(RocksDbWeight::get().reads(2_u64)) + .saturating_add(RocksDbWeight::get().writes(1_u64)) + } + fn on_init_loop() -> Weight { + // Proof Size summary in bytes: + // Measured: `0` + // Estimated: `0` + // Minimum execution time: 440_000 picoseconds. + Weight::from_parts(500_000, 0) + } + /// Storage: `MultiBlockMigrations::Cursor` (r:0 w:1) + /// Proof: `MultiBlockMigrations::Cursor` (`max_values`: Some(1), `max_size`: Some(65550), added: 66045, mode: `MaxEncodedLen`) + fn force_set_cursor() -> Weight { + // Proof Size summary in bytes: + // Measured: `0` + // Estimated: `0` + // Minimum execution time: 5_751_000 picoseconds. + Weight::from_parts(5_950_000, 0) + .saturating_add(RocksDbWeight::get().writes(1_u64)) + } + /// Storage: `MultiBlockMigrations::Cursor` (r:0 w:1) + /// Proof: `MultiBlockMigrations::Cursor` (`max_values`: Some(1), `max_size`: Some(65550), added: 66045, mode: `MaxEncodedLen`) + fn force_set_active_cursor() -> Weight { + // Proof Size summary in bytes: + // Measured: `0` + // Estimated: `0` + // Minimum execution time: 6_350_000 picoseconds. + Weight::from_parts(6_560_000, 0) + .saturating_add(RocksDbWeight::get().writes(1_u64)) + } + /// Storage: `MultiBlockMigrations::Cursor` (r:1 w:0) + /// Proof: `MultiBlockMigrations::Cursor` (`max_values`: Some(1), `max_size`: Some(65550), added: 66045, mode: `MaxEncodedLen`) + /// Storage: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Proof: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + fn force_onboard_mbms() -> Weight { + // Proof Size summary in bytes: + // Measured: `218` + // Estimated: `67035` + // Minimum execution time: 11_121_000 picoseconds. + Weight::from_parts(11_530_000, 67035) + .saturating_add(RocksDbWeight::get().reads(2_u64)) + } + /// Storage: `MultiBlockMigrations::Historic` (r:256 w:256) + /// Proof: `MultiBlockMigrations::Historic` (`max_values`: None, `max_size`: Some(266), added: 2741, mode: `MaxEncodedLen`) + /// The range of component `n` is `[0, 256]`. + fn clear_historic(n: u32, ) -> Weight { + // Proof Size summary in bytes: + // Measured: `1089 + n * (271 ±0)` + // Estimated: `3834 + n * (2740 ±0)` + // Minimum execution time: 21_891_000 picoseconds. + Weight::from_parts(18_572_306, 3834) + // Standard Error: 3_236 + .saturating_add(Weight::from_parts(1_648_429, 0).saturating_mul(n.into())) + .saturating_add(RocksDbWeight::get().reads(1_u64)) + .saturating_add(RocksDbWeight::get().reads((1_u64).saturating_mul(n.into()))) + .saturating_add(RocksDbWeight::get().writes((1_u64).saturating_mul(n.into()))) + .saturating_add(Weight::from_parts(0, 2740).saturating_mul(n.into())) + } +} diff --git a/substrate/frame/src/lib.rs b/substrate/frame/src/lib.rs index a2e5d726fdd0..52db7c34bfdc 100644 --- a/substrate/frame/src/lib.rs +++ b/substrate/frame/src/lib.rs @@ -197,7 +197,7 @@ pub mod runtime { // Types often used in the runtime APIs. pub use sp_core::OpaqueMetadata; pub use sp_inherents::{CheckInherentsResult, InherentData}; - pub use sp_runtime::ApplyExtrinsicResult; + pub use sp_runtime::{ApplyExtrinsicResult, ExtrinsicInclusionMode}; pub use frame_system_rpc_runtime_api::*; pub use sp_api::{self, *}; diff --git a/substrate/frame/support/Cargo.toml b/substrate/frame/support/Cargo.toml index 72e2d0bfa569..c19f478c803e 100644 --- a/substrate/frame/support/Cargo.toml +++ b/substrate/frame/support/Cargo.toml @@ -18,13 +18,24 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] array-bytes = { version = "6.1", default-features = false } serde = { features = ["alloc", "derive"], workspace = true } -codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = ["derive", "max-encoded-len"] } -scale-info = { version = "2.10.0", default-features = false, features = ["derive"] } -frame-metadata = { version = "16.0.0", default-features = false, features = ["current"] } -sp-api = { path = "../../primitives/api", default-features = false, features = ["frame-metadata"] } +codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = [ + "derive", + "max-encoded-len", +] } +scale-info = { version = "2.10.0", default-features = false, features = [ + "derive", +] } +frame-metadata = { version = "16.0.0", default-features = false, features = [ + "current", +] } +sp-api = { path = "../../primitives/api", default-features = false, features = [ + "frame-metadata", +] } sp-std = { path = "../../primitives/std", default-features = false } sp-io = { path = "../../primitives/io", default-features = false } -sp-runtime = { path = "../../primitives/runtime", default-features = false, features = ["serde"] } +sp-runtime = { path = "../../primitives/runtime", default-features = false, features = [ + "serde", +] } sp-tracing = { path = "../../primitives/tracing", default-features = false } sp-core = { path = "../../primitives/core", default-features = false } sp-arithmetic = { path = "../../primitives/arithmetic", default-features = false } diff --git a/substrate/frame/support/procedural/src/construct_runtime/expand/inherent.rs b/substrate/frame/support/procedural/src/construct_runtime/expand/inherent.rs index 34b9d21d8ce8..da483fa6cf0b 100644 --- a/substrate/frame/support/procedural/src/construct_runtime/expand/inherent.rs +++ b/substrate/frame/support/procedural/src/construct_runtime/expand/inherent.rs @@ -204,47 +204,50 @@ pub fn expand_outer_inherent( } } + impl #scrate::traits::IsInherent<<#block as #scrate::sp_runtime::traits::Block>::Extrinsic> for #runtime { + fn is_inherent(ext: &<#block as #scrate::sp_runtime::traits::Block>::Extrinsic) -> bool { + use #scrate::inherent::ProvideInherent; + use #scrate::traits::{IsSubType, ExtrinsicCall}; + + if #scrate::sp_runtime::traits::Extrinsic::is_signed(ext).unwrap_or(false) { + // Signed extrinsics are never inherents. + return false + } + + #( + #pallet_attrs + { + let call = <#unchecked_extrinsic as ExtrinsicCall>::call(ext); + if let Some(call) = IsSubType::<_>::is_sub_type(call) { + if <#pallet_names as ProvideInherent>::is_inherent(&call) { + return true; + } + } + } + )* + false + } + } + impl #scrate::traits::EnsureInherentsAreFirst<#block> for #runtime { - fn ensure_inherents_are_first(block: &#block) -> Result<(), u32> { + fn ensure_inherents_are_first(block: &#block) -> Result { use #scrate::inherent::ProvideInherent; use #scrate::traits::{IsSubType, ExtrinsicCall}; use #scrate::sp_runtime::traits::Block as _; - let mut first_signed_observed = false; + let mut num_inherents = 0u32; for (i, xt) in block.extrinsics().iter().enumerate() { - let is_signed = #scrate::sp_runtime::traits::Extrinsic::is_signed(xt) - .unwrap_or(false); - - let is_inherent = if is_signed { - // Signed extrinsics are not inherents. - false - } else { - let mut is_inherent = false; - #( - #pallet_attrs - { - let call = <#unchecked_extrinsic as ExtrinsicCall>::call(xt); - if let Some(call) = IsSubType::<_>::is_sub_type(call) { - if #pallet_names::is_inherent(&call) { - is_inherent = true; - } - } - } - )* - is_inherent - }; - - if !is_inherent { - first_signed_observed = true; - } + if >::is_inherent(xt) { + if num_inherents != i as u32 { + return Err(i as u32); + } - if first_signed_observed && is_inherent { - return Err(i as u32) + num_inherents += 1; // Safe since we are in an `enumerate` loop. } } - Ok(()) + Ok(num_inherents) } } } diff --git a/substrate/frame/support/procedural/src/pallet/expand/call.rs b/substrate/frame/support/procedural/src/pallet/expand/call.rs index f43faba1ee0c..f395872c8a80 100644 --- a/substrate/frame/support/procedural/src/pallet/expand/call.rs +++ b/substrate/frame/support/procedural/src/pallet/expand/call.rs @@ -18,7 +18,7 @@ use crate::{ pallet::{ expand::warnings::{weight_constant_warning, weight_witness_warning}, - parse::call::{CallVariantDef, CallWeightDef}, + parse::call::CallWeightDef, Def, }, COUNTER, @@ -112,22 +112,7 @@ pub fn expand_call(def: &mut Def) -> proc_macro2::TokenStream { } debug_assert_eq!(fn_weight.len(), methods.len()); - let map_fn_docs = if !def.dev_mode { - // Emit the [`Pallet::method`] documentation only for non-dev modes. - |method: &CallVariantDef| { - let reference = format!("See [`Pallet::{}`].", method.name); - quote!(#reference) - } - } else { - // For the dev-mode do not provide a documenation link as it will break the - // `cargo doc` if the pallet is private inside a test. - |method: &CallVariantDef| { - let reference = format!("See `Pallet::{}`.", method.name); - quote!(#reference) - } - }; - - let fn_doc = methods.iter().map(map_fn_docs).collect::>(); + let fn_doc = methods.iter().map(|method| &method.docs).collect::>(); let args_name = methods .iter() @@ -309,7 +294,7 @@ pub fn expand_call(def: &mut Def) -> proc_macro2::TokenStream { ), #( #cfg_attrs - #[doc = #fn_doc] + #( #[doc = #fn_doc] )* #[codec(index = #call_index)] #fn_name { #( diff --git a/substrate/frame/support/procedural/src/pallet/expand/hooks.rs b/substrate/frame/support/procedural/src/pallet/expand/hooks.rs index 6b25ddcba1a7..3623b595268d 100644 --- a/substrate/frame/support/procedural/src/pallet/expand/hooks.rs +++ b/substrate/frame/support/procedural/src/pallet/expand/hooks.rs @@ -175,6 +175,22 @@ pub fn expand_hooks(def: &mut Def) -> proc_macro2::TokenStream { } } + impl<#type_impl_gen> + #frame_support::traits::OnPoll<#frame_system::pallet_prelude::BlockNumberFor::> + for #pallet_ident<#type_use_gen> #where_clause + { + fn on_poll( + n: #frame_system::pallet_prelude::BlockNumberFor::, + weight: &mut #frame_support::weights::WeightMeter + ) { + < + Self as #frame_support::traits::Hooks< + #frame_system::pallet_prelude::BlockNumberFor:: + > + >::on_poll(n, weight); + } + } + impl<#type_impl_gen> #frame_support::traits::OnInitialize<#frame_system::pallet_prelude::BlockNumberFor::> for #pallet_ident<#type_use_gen> #where_clause diff --git a/substrate/frame/support/src/migrations.rs b/substrate/frame/support/src/migrations.rs index f75410590232..2ceab44cb16b 100644 --- a/substrate/frame/support/src/migrations.rs +++ b/substrate/frame/support/src/migrations.rs @@ -16,13 +16,21 @@ // limitations under the License. use crate::{ - traits::{GetStorageVersion, NoStorageVersionSet, PalletInfoAccess, StorageVersion}, - weights::{RuntimeDbWeight, Weight}, + defensive, + storage::transactional::with_transaction_opaque_err, + traits::{ + Defensive, GetStorageVersion, NoStorageVersionSet, PalletInfoAccess, SafeMode, + StorageVersion, + }, + weights::{RuntimeDbWeight, Weight, WeightMeter}, }; +use codec::{Decode, Encode, MaxEncodedLen}; use impl_trait_for_tuples::impl_for_tuples; +use sp_arithmetic::traits::Bounded; use sp_core::Get; use sp_io::{hashing::twox_128, storage::clear_prefix, KillStorageResult}; -use sp_std::marker::PhantomData; +use sp_runtime::traits::Zero; +use sp_std::{marker::PhantomData, vec::Vec}; /// Handles storage migration pallet versioning. /// @@ -91,7 +99,7 @@ pub struct VersionedMigration), @@ -118,7 +126,6 @@ impl< /// migration ran or not. #[cfg(feature = "try-runtime")] fn pre_upgrade() -> Result, sp_runtime::TryRuntimeError> { - use codec::Encode; let on_chain_version = Pallet::on_chain_storage_version(); if on_chain_version == FROM { Ok(VersionedPostUpgradeData::MigrationExecuted(Inner::pre_upgrade()?).encode()) @@ -361,3 +368,622 @@ impl, DbWeight: Get> frame_support::traits Ok(()) } } + +/// A migration that can proceed in multiple steps. +pub trait SteppedMigration { + /// The cursor type that stores the progress (aka. state) of this migration. + type Cursor: codec::FullCodec + codec::MaxEncodedLen; + + /// The unique identifier type of this migration. + type Identifier: codec::FullCodec + codec::MaxEncodedLen; + + /// The unique identifier of this migration. + /// + /// If two migrations have the same identifier, then they are assumed to be identical. + fn id() -> Self::Identifier; + + /// The maximum number of steps that this migration can take. + /// + /// This can be used to enforce progress and prevent migrations becoming stuck forever. A + /// migration that exceeds its max steps is treated as failed. `None` means that there is no + /// limit. + fn max_steps() -> Option { + None + } + + /// Try to migrate as much as possible with the given weight. + /// + /// **ANY STORAGE CHANGES MUST BE ROLLED-BACK BY THE CALLER UPON ERROR.** This is necessary + /// since the caller cannot return a cursor in the error case. [`Self::transactional_step`] is + /// provided as convenience for a caller. A cursor of `None` implies that the migration is at + /// its end. A migration that once returned `Nonce` is guaranteed to never be called again. + fn step( + cursor: Option, + meter: &mut WeightMeter, + ) -> Result, SteppedMigrationError>; + + /// Same as [`Self::step`], but rolls back pending changes in the error case. + fn transactional_step( + mut cursor: Option, + meter: &mut WeightMeter, + ) -> Result, SteppedMigrationError> { + with_transaction_opaque_err(move || match Self::step(cursor, meter) { + Ok(new_cursor) => { + cursor = new_cursor; + sp_runtime::TransactionOutcome::Commit(Ok(cursor)) + }, + Err(err) => sp_runtime::TransactionOutcome::Rollback(Err(err)), + }) + .map_err(|()| SteppedMigrationError::Failed)? + } +} + +/// Error that can occur during a [`SteppedMigration`]. +#[derive(Debug, Encode, Decode, MaxEncodedLen, scale_info::TypeInfo)] +pub enum SteppedMigrationError { + // Transient errors: + /// The remaining weight is not enough to do anything. + /// + /// Can be resolved by calling with at least `required` weight. Note that calling it with + /// exactly `required` weight could cause it to not make any progress. + InsufficientWeight { + /// Amount of weight required to make progress. + required: Weight, + }, + // Permanent errors: + /// The migration cannot decode its cursor and therefore not proceed. + /// + /// This should not happen unless (1) the migration itself returned an invalid cursor in a + /// previous iteration, (2) the storage got corrupted or (3) there is a bug in the caller's + /// code. + InvalidCursor, + /// The migration encountered a permanent error and cannot continue. + Failed, +} + +/// Notification handler for status updates regarding Multi-Block-Migrations. +#[impl_trait_for_tuples::impl_for_tuples(8)] +pub trait MigrationStatusHandler { + /// Notifies of the start of a runtime migration. + fn started() {} + + /// Notifies of the completion of a runtime migration. + fn completed() {} +} + +/// Handles a failed runtime migration. +/// +/// This should never happen, but is here for completeness. +pub trait FailedMigrationHandler { + /// Infallibly handle a failed runtime migration. + /// + /// Gets passed in the optional index of the migration in the batch that caused the failure. + /// Returning `None` means that no automatic handling should take place and the callee decides + /// in the implementation what to do. + fn failed(migration: Option) -> FailedMigrationHandling; +} + +/// Do now allow any transactions to be processed after a runtime upgrade failed. +/// +/// This is **not a sane default**, since it prevents governance intervention. +pub struct FreezeChainOnFailedMigration; + +impl FailedMigrationHandler for FreezeChainOnFailedMigration { + fn failed(_migration: Option) -> FailedMigrationHandling { + FailedMigrationHandling::KeepStuck + } +} + +/// Enter safe mode on a failed runtime upgrade. +/// +/// This can be very useful to manually intervene and fix the chain state. `Else` is used in case +/// that the safe mode could not be entered. +pub struct EnterSafeModeOnFailedMigration( + PhantomData<(SM, Else)>, +); + +impl FailedMigrationHandler + for EnterSafeModeOnFailedMigration +where + ::BlockNumber: Bounded, +{ + fn failed(migration: Option) -> FailedMigrationHandling { + let entered = if SM::is_entered() { + SM::extend(Bounded::max_value()) + } else { + SM::enter(Bounded::max_value()) + }; + + // If we could not enter or extend safe mode (for whatever reason), then we try the next. + if entered.is_err() { + Else::failed(migration) + } else { + FailedMigrationHandling::KeepStuck + } + } +} + +/// How to proceed after a runtime upgrade failed. +/// +/// There is NO SANE DEFAULT HERE. All options are very dangerous and should be used with care. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FailedMigrationHandling { + /// Resume extrinsic processing of the chain. This will not resume the upgrade. + /// + /// This should be supplemented with additional measures to ensure that the broken chain state + /// does not get further messed up by user extrinsics. + ForceUnstuck, + /// Set the cursor to `Stuck` and keep blocking extrinsics. + KeepStuck, + /// Don't do anything with the cursor and let the handler decide. + /// + /// This can be useful in cases where the other two options would overwrite any changes that + /// were done by the handler to the cursor. + Ignore, +} + +/// Something that can do multi step migrations. +pub trait MultiStepMigrator { + /// Hint for whether [`Self::step`] should be called. + fn ongoing() -> bool; + + /// Do the next step in the MBM process. + /// + /// Must gracefully handle the case that it is currently not upgrading. + fn step() -> Weight; +} + +impl MultiStepMigrator for () { + fn ongoing() -> bool { + false + } + + fn step() -> Weight { + Weight::zero() + } +} + +/// Multiple [`SteppedMigration`]. +pub trait SteppedMigrations { + /// The number of migrations that `Self` aggregates. + fn len() -> u32; + + /// The `n`th [`SteppedMigration::id`]. + /// + /// Is guaranteed to return `Some` if `n < Self::len()`. + fn nth_id(n: u32) -> Option>; + + /// The [`SteppedMigration::max_steps`] of the `n`th migration. + /// + /// Is guaranteed to return `Some` if `n < Self::len()`. + fn nth_max_steps(n: u32) -> Option>; + + /// Do a [`SteppedMigration::step`] on the `n`th migration. + /// + /// Is guaranteed to return `Some` if `n < Self::len()`. + fn nth_step( + n: u32, + cursor: Option>, + meter: &mut WeightMeter, + ) -> Option>, SteppedMigrationError>>; + + /// Do a [`SteppedMigration::transactional_step`] on the `n`th migration. + /// + /// Is guaranteed to return `Some` if `n < Self::len()`. + fn nth_transactional_step( + n: u32, + cursor: Option>, + meter: &mut WeightMeter, + ) -> Option>, SteppedMigrationError>>; + + /// The maximal encoded length across all cursors. + fn cursor_max_encoded_len() -> usize; + + /// The maximal encoded length across all identifiers. + fn identifier_max_encoded_len() -> usize; + + /// Assert the integrity of the migrations. + /// + /// Should be executed as part of a test prior to runtime usage. May or may not need + /// externalities. + #[cfg(feature = "std")] + fn integrity_test() -> Result<(), &'static str> { + use crate::ensure; + let l = Self::len(); + + for n in 0..l { + ensure!(Self::nth_id(n).is_some(), "id is None"); + ensure!(Self::nth_max_steps(n).is_some(), "steps is None"); + + // The cursor that we use does not matter. Hence use empty. + ensure!( + Self::nth_step(n, Some(vec![]), &mut WeightMeter::new()).is_some(), + "steps is None" + ); + ensure!( + Self::nth_transactional_step(n, Some(vec![]), &mut WeightMeter::new()).is_some(), + "steps is None" + ); + } + + Ok(()) + } +} + +impl SteppedMigrations for () { + fn len() -> u32 { + 0 + } + + fn nth_id(_n: u32) -> Option> { + None + } + + fn nth_max_steps(_n: u32) -> Option> { + None + } + + fn nth_step( + _n: u32, + _cursor: Option>, + _meter: &mut WeightMeter, + ) -> Option>, SteppedMigrationError>> { + None + } + + fn nth_transactional_step( + _n: u32, + _cursor: Option>, + _meter: &mut WeightMeter, + ) -> Option>, SteppedMigrationError>> { + None + } + + fn cursor_max_encoded_len() -> usize { + 0 + } + + fn identifier_max_encoded_len() -> usize { + 0 + } +} + +// A collection consisting of only a single migration. +impl SteppedMigrations for T { + fn len() -> u32 { + 1 + } + + fn nth_id(_n: u32) -> Option> { + Some(T::id().encode()) + } + + fn nth_max_steps(n: u32) -> Option> { + // It should be generally fine to call with n>0, but the code should not attempt to. + n.is_zero() + .then_some(T::max_steps()) + .defensive_proof("nth_max_steps should only be called with n==0") + } + + fn nth_step( + _n: u32, + cursor: Option>, + meter: &mut WeightMeter, + ) -> Option>, SteppedMigrationError>> { + if !_n.is_zero() { + defensive!("nth_step should only be called with n==0"); + return None + } + + let cursor = match cursor { + Some(cursor) => match T::Cursor::decode(&mut &cursor[..]) { + Ok(cursor) => Some(cursor), + Err(_) => return Some(Err(SteppedMigrationError::InvalidCursor)), + }, + None => None, + }; + + Some(T::step(cursor, meter).map(|cursor| cursor.map(|cursor| cursor.encode()))) + } + + fn nth_transactional_step( + n: u32, + cursor: Option>, + meter: &mut WeightMeter, + ) -> Option>, SteppedMigrationError>> { + if n != 0 { + defensive!("nth_transactional_step should only be called with n==0"); + return None + } + + let cursor = match cursor { + Some(cursor) => match T::Cursor::decode(&mut &cursor[..]) { + Ok(cursor) => Some(cursor), + Err(_) => return Some(Err(SteppedMigrationError::InvalidCursor)), + }, + None => None, + }; + + Some( + T::transactional_step(cursor, meter).map(|cursor| cursor.map(|cursor| cursor.encode())), + ) + } + + fn cursor_max_encoded_len() -> usize { + T::Cursor::max_encoded_len() + } + + fn identifier_max_encoded_len() -> usize { + T::Identifier::max_encoded_len() + } +} + +#[impl_trait_for_tuples::impl_for_tuples(1, 30)] +impl SteppedMigrations for Tuple { + fn len() -> u32 { + for_tuples!( #( Tuple::len() )+* ) + } + + fn nth_id(n: u32) -> Option> { + let mut i = 0; + + for_tuples!( #( + if (i + Tuple::len()) > n { + return Tuple::nth_id(n - i) + } + + i += Tuple::len(); + )* ); + + None + } + + fn nth_step( + n: u32, + cursor: Option>, + meter: &mut WeightMeter, + ) -> Option>, SteppedMigrationError>> { + let mut i = 0; + + for_tuples!( #( + if (i + Tuple::len()) > n { + return Tuple::nth_step(n - i, cursor, meter) + } + + i += Tuple::len(); + )* ); + + None + } + + fn nth_transactional_step( + n: u32, + cursor: Option>, + meter: &mut WeightMeter, + ) -> Option>, SteppedMigrationError>> { + let mut i = 0; + + for_tuples! ( #( + if (i + Tuple::len()) > n { + return Tuple::nth_transactional_step(n - i, cursor, meter) + } + + i += Tuple::len(); + )* ); + + None + } + + fn nth_max_steps(n: u32) -> Option> { + let mut i = 0; + + for_tuples!( #( + if (i + Tuple::len()) > n { + return Tuple::nth_max_steps(n - i) + } + + i += Tuple::len(); + )* ); + + None + } + + fn cursor_max_encoded_len() -> usize { + let mut max_len = 0; + + for_tuples!( #( + max_len = max_len.max(Tuple::cursor_max_encoded_len()); + )* ); + + max_len + } + + fn identifier_max_encoded_len() -> usize { + let mut max_len = 0; + + for_tuples!( #( + max_len = max_len.max(Tuple::identifier_max_encoded_len()); + )* ); + + max_len + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{assert_ok, storage::unhashed}; + + #[derive(Decode, Encode, MaxEncodedLen, Eq, PartialEq)] + pub enum Either { + Left(L), + Right(R), + } + + pub struct M0; + impl SteppedMigration for M0 { + type Cursor = (); + type Identifier = u8; + + fn id() -> Self::Identifier { + 0 + } + + fn step( + _cursor: Option, + _meter: &mut WeightMeter, + ) -> Result, SteppedMigrationError> { + log::info!("M0"); + unhashed::put(&[0], &()); + Ok(None) + } + } + + pub struct M1; + impl SteppedMigration for M1 { + type Cursor = (); + type Identifier = u8; + + fn id() -> Self::Identifier { + 1 + } + + fn step( + _cursor: Option, + _meter: &mut WeightMeter, + ) -> Result, SteppedMigrationError> { + log::info!("M1"); + unhashed::put(&[1], &()); + Ok(None) + } + + fn max_steps() -> Option { + Some(1) + } + } + + pub struct M2; + impl SteppedMigration for M2 { + type Cursor = (); + type Identifier = u8; + + fn id() -> Self::Identifier { + 2 + } + + fn step( + _cursor: Option, + _meter: &mut WeightMeter, + ) -> Result, SteppedMigrationError> { + log::info!("M2"); + unhashed::put(&[2], &()); + Ok(None) + } + + fn max_steps() -> Option { + Some(2) + } + } + + pub struct F0; + impl SteppedMigration for F0 { + type Cursor = (); + type Identifier = u8; + + fn id() -> Self::Identifier { + 3 + } + + fn step( + _cursor: Option, + _meter: &mut WeightMeter, + ) -> Result, SteppedMigrationError> { + log::info!("F0"); + unhashed::put(&[3], &()); + Err(SteppedMigrationError::Failed) + } + } + + // Three migrations combined to execute in order: + type Triple = (M0, (M1, M2)); + // Six migrations, just concatenating the ones from before: + type Hextuple = (Triple, Triple); + + #[test] + fn singular_migrations_work() { + assert_eq!(M0::max_steps(), None); + assert_eq!(M1::max_steps(), Some(1)); + assert_eq!(M2::max_steps(), Some(2)); + + assert_eq!(<(M0, M1)>::nth_max_steps(0), Some(None)); + assert_eq!(<(M0, M1)>::nth_max_steps(1), Some(Some(1))); + assert_eq!(<(M0, M1, M2)>::nth_max_steps(2), Some(Some(2))); + + assert_eq!(<(M0, M1)>::nth_max_steps(2), None); + } + + #[test] + fn tuple_migrations_work() { + assert_eq!(<() as SteppedMigrations>::len(), 0); + assert_eq!(<((), ((), ())) as SteppedMigrations>::len(), 0); + assert_eq!(::len(), 3); + assert_eq!(::len(), 6); + + // Check the IDs. The index specific functions all return an Option, + // to account for the out-of-range case. + assert_eq!(::nth_id(0), Some(0u8.encode())); + assert_eq!(::nth_id(1), Some(1u8.encode())); + assert_eq!(::nth_id(2), Some(2u8.encode())); + + sp_io::TestExternalities::default().execute_with(|| { + for n in 0..3 { + ::nth_step( + n, + Default::default(), + &mut WeightMeter::new(), + ); + } + }); + } + + #[test] + fn integrity_test_works() { + sp_io::TestExternalities::default().execute_with(|| { + assert_ok!(<() as SteppedMigrations>::integrity_test()); + assert_ok!(::integrity_test()); + assert_ok!(::integrity_test()); + assert_ok!(::integrity_test()); + assert_ok!(::integrity_test()); + assert_ok!(::integrity_test()); + }); + } + + #[test] + fn transactional_rollback_works() { + sp_io::TestExternalities::default().execute_with(|| { + assert_ok!(<(M0, F0) as SteppedMigrations>::nth_transactional_step( + 0, + Default::default(), + &mut WeightMeter::new() + ) + .unwrap()); + assert!(unhashed::exists(&[0])); + + let _g = crate::StorageNoopGuard::new(); + assert!(<(M0, F0) as SteppedMigrations>::nth_transactional_step( + 1, + Default::default(), + &mut WeightMeter::new() + ) + .unwrap() + .is_err()); + assert!(<(F0, M1) as SteppedMigrations>::nth_transactional_step( + 0, + Default::default(), + &mut WeightMeter::new() + ) + .unwrap() + .is_err()); + }); + } +} diff --git a/substrate/frame/support/src/storage/transactional.rs b/substrate/frame/support/src/storage/transactional.rs index d42e1809e912..0671db4a3a86 100644 --- a/substrate/frame/support/src/storage/transactional.rs +++ b/substrate/frame/support/src/storage/transactional.rs @@ -127,6 +127,22 @@ where } } +/// Same as [`with_transaction`] but casts any internal error to `()`. +/// +/// This rids `E` of the `From` bound that is required by `with_transaction`. +pub fn with_transaction_opaque_err(f: F) -> Result, ()> +where + F: FnOnce() -> TransactionOutcome>, +{ + with_transaction(move || -> TransactionOutcome, DispatchError>> { + match f() { + TransactionOutcome::Commit(res) => TransactionOutcome::Commit(Ok(res)), + TransactionOutcome::Rollback(res) => TransactionOutcome::Rollback(Ok(res)), + } + }) + .map_err(|_| ()) +} + /// Same as [`with_transaction`] but without a limit check on nested transactional layers. /// /// This is mostly for backwards compatibility before there was a transactional layer limit. diff --git a/substrate/frame/support/src/traits.rs b/substrate/frame/support/src/traits.rs index 3d0429f71b11..1997d8fc223e 100644 --- a/substrate/frame/support/src/traits.rs +++ b/substrate/frame/support/src/traits.rs @@ -59,10 +59,10 @@ pub use misc::{ AccountTouch, Backing, ConstBool, ConstI128, ConstI16, ConstI32, ConstI64, ConstI8, ConstU128, ConstU16, ConstU32, ConstU64, ConstU8, DefensiveMax, DefensiveMin, DefensiveSaturating, DefensiveTruncateFrom, EnsureInherentsAreFirst, EqualPrivilegeOnly, EstimateCallFee, - ExecuteBlock, ExtrinsicCall, Get, GetBacking, GetDefault, HandleLifetime, IsSubType, IsType, - Len, OffchainWorker, OnKilledAccount, OnNewAccount, PrivilegeCmp, SameOrOther, Time, - TryCollect, TryDrop, TypedGet, UnixTime, VariantCount, VariantCountOf, WrapperKeepOpaque, - WrapperOpaque, + ExecuteBlock, ExtrinsicCall, Get, GetBacking, GetDefault, HandleLifetime, IsInherent, + IsSubType, IsType, Len, OffchainWorker, OnKilledAccount, OnNewAccount, PrivilegeCmp, + SameOrOther, Time, TryCollect, TryDrop, TypedGet, UnixTime, VariantCount, VariantCountOf, + WrapperKeepOpaque, WrapperOpaque, }; #[allow(deprecated)] pub use misc::{PreimageProvider, PreimageRecipient}; @@ -86,7 +86,8 @@ mod hooks; pub use hooks::GenesisBuild; pub use hooks::{ BeforeAllRuntimeMigrations, BuildGenesisConfig, Hooks, IntegrityTest, OnFinalize, OnGenesis, - OnIdle, OnInitialize, OnRuntimeUpgrade, OnTimestampSet, + OnIdle, OnInitialize, OnPoll, OnRuntimeUpgrade, OnTimestampSet, PostInherents, + PostTransactions, PreInherents, }; pub mod schedule; diff --git a/substrate/frame/support/src/traits/hooks.rs b/substrate/frame/support/src/traits/hooks.rs index 72f047c3a73d..7d0e5aa1e89c 100644 --- a/substrate/frame/support/src/traits/hooks.rs +++ b/substrate/frame/support/src/traits/hooks.rs @@ -25,10 +25,74 @@ use crate::weights::Weight; use impl_trait_for_tuples::impl_for_tuples; use sp_runtime::traits::AtLeast32BitUnsigned; use sp_std::prelude::*; +use sp_weights::WeightMeter; #[cfg(feature = "try-runtime")] use sp_runtime::TryRuntimeError; +/// Provides a callback to execute logic before the all inherents. +pub trait PreInherents { + /// Called before all inherents were applied but after `on_initialize`. + fn pre_inherents() {} +} + +#[cfg_attr(all(not(feature = "tuples-96"), not(feature = "tuples-128")), impl_for_tuples(64))] +#[cfg_attr(all(feature = "tuples-96", not(feature = "tuples-128")), impl_for_tuples(96))] +#[cfg_attr(feature = "tuples-128", impl_for_tuples(128))] +impl PreInherents for Tuple { + fn pre_inherents() { + for_tuples!( #( Tuple::pre_inherents(); )* ); + } +} + +/// Provides a callback to execute logic after the all inherents. +pub trait PostInherents { + /// Called after all inherents were applied. + fn post_inherents() {} +} + +#[cfg_attr(all(not(feature = "tuples-96"), not(feature = "tuples-128")), impl_for_tuples(64))] +#[cfg_attr(all(feature = "tuples-96", not(feature = "tuples-128")), impl_for_tuples(96))] +#[cfg_attr(feature = "tuples-128", impl_for_tuples(128))] +impl PostInherents for Tuple { + fn post_inherents() { + for_tuples!( #( Tuple::post_inherents(); )* ); + } +} + +/// Provides a callback to execute logic before the all transactions. +pub trait PostTransactions { + /// Called after all transactions were applied but before `on_finalize`. + fn post_transactions() {} +} + +#[cfg_attr(all(not(feature = "tuples-96"), not(feature = "tuples-128")), impl_for_tuples(64))] +#[cfg_attr(all(feature = "tuples-96", not(feature = "tuples-128")), impl_for_tuples(96))] +#[cfg_attr(feature = "tuples-128", impl_for_tuples(128))] +impl PostTransactions for Tuple { + fn post_transactions() { + for_tuples!( #( Tuple::post_transactions(); )* ); + } +} + +/// Periodically executes logic. Is not guaranteed to run within a specific timeframe and should +/// only be used on logic that has no deadline. +pub trait OnPoll { + /// Code to execute every now and then at the beginning of the block after inherent application. + /// + /// The remaining weight limit must be respected. + fn on_poll(_n: BlockNumber, _weight: &mut WeightMeter) {} +} + +#[cfg_attr(all(not(feature = "tuples-96"), not(feature = "tuples-128")), impl_for_tuples(64))] +#[cfg_attr(all(feature = "tuples-96", not(feature = "tuples-128")), impl_for_tuples(96))] +#[cfg_attr(feature = "tuples-128", impl_for_tuples(128))] +impl OnPoll for Tuple { + fn on_poll(n: BlockNumber, weight: &mut WeightMeter) { + for_tuples!( #( Tuple::on_poll(n.clone(), weight); )* ); + } +} + /// See [`Hooks::on_initialize`]. pub trait OnInitialize { /// See [`Hooks::on_initialize`]. @@ -374,6 +438,12 @@ pub trait Hooks { Weight::zero() } + /// A hook to run logic after inherent application. + /// + /// Is not guaranteed to execute in a block and should therefore only be used in no-deadline + /// scenarios. + fn on_poll(_n: BlockNumber, _weight: &mut WeightMeter) {} + /// Hook executed when a code change (aka. a "runtime upgrade") is detected by the FRAME /// `Executive` pallet. /// @@ -446,7 +516,7 @@ pub trait Hooks { } /// Implementing this function on a pallet allows you to perform long-running tasks that are - /// dispatched as separate threads, and entirely independent of the main wasm runtime. + /// dispatched as separate threads, and entirely independent of the main blockchain execution. /// /// This function can freely read from the state, but any change it makes to the state is /// meaningless. Writes can be pushed back to the chain by submitting extrinsics from the diff --git a/substrate/frame/support/src/traits/misc.rs b/substrate/frame/support/src/traits/misc.rs index eafd9c8abdd2..1f634a642829 100644 --- a/substrate/frame/support/src/traits/misc.rs +++ b/substrate/frame/support/src/traits/misc.rs @@ -23,6 +23,7 @@ use impl_trait_for_tuples::impl_for_tuples; use scale_info::{build::Fields, meta_type, Path, Type, TypeInfo, TypeParameter}; use sp_arithmetic::traits::{CheckedAdd, CheckedMul, CheckedSub, One, Saturating}; use sp_core::bounded::bounded_vec::TruncateFrom; + #[doc(hidden)] pub use sp_runtime::traits::{ ConstBool, ConstI128, ConstI16, ConstI32, ConstI64, ConstI8, ConstU128, ConstU16, ConstU32, @@ -895,11 +896,21 @@ pub trait GetBacking { /// A trait to ensure the inherent are before non-inherent in a block. /// /// This is typically implemented on runtime, through `construct_runtime!`. -pub trait EnsureInherentsAreFirst { +pub trait EnsureInherentsAreFirst: + IsInherent<::Extrinsic> +{ /// Ensure the position of inherent is correct, i.e. they are before non-inherents. /// - /// On error return the index of the inherent with invalid position (counting from 0). - fn ensure_inherents_are_first(block: &Block) -> Result<(), u32>; + /// On error return the index of the inherent with invalid position (counting from 0). On + /// success it returns the index of the last inherent. `0` therefore means that there are no + /// inherents. + fn ensure_inherents_are_first(block: &Block) -> Result; +} + +/// A trait to check if an extrinsic is an inherent. +pub trait IsInherent { + /// Whether this extrinsic is an inherent. + fn is_inherent(ext: &Extrinsic) -> bool; } /// An extrinsic on which we can get access to call. diff --git a/substrate/frame/support/test/tests/construct_runtime_ui/undefined_inherent_part.stderr b/substrate/frame/support/test/tests/construct_runtime_ui/undefined_inherent_part.stderr index a4c7ecf78658..30005c07cb63 100644 --- a/substrate/frame/support/test/tests/construct_runtime_ui/undefined_inherent_part.stderr +++ b/substrate/frame/support/test/tests/construct_runtime_ui/undefined_inherent_part.stderr @@ -53,8 +53,9 @@ error[E0599]: no function or associated item named `is_inherent` found for struc | |_^ function or associated item not found in `Pallet` | = help: items from traits can only be used if the trait is implemented and in scope - = note: the following trait defines an item `is_inherent`, perhaps you need to implement it: + = note: the following traits define an item `is_inherent`, perhaps you need to implement one of them: candidate #1: `ProvideInherent` + candidate #2: `IsInherent` = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) error[E0599]: no function or associated item named `check_inherent` found for struct `pallet::Pallet` in the current scope @@ -119,3 +120,23 @@ error[E0599]: no function or associated item named `is_inherent_required` found = note: the following trait defines an item `is_inherent_required`, perhaps you need to implement it: candidate #1: `ProvideInherent` = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0277]: the trait bound `pallet::Pallet: ProvideInherent` is not satisfied + --> tests/construct_runtime_ui/undefined_inherent_part.rs:70:3 + | +70 | Pallet: pallet expanded::{}::{Pallet, Inherent}, + | ^^^^^^ the trait `ProvideInherent` is not implemented for `pallet::Pallet` + +error[E0277]: the trait bound `pallet::Pallet: ProvideInherent` is not satisfied + --> tests/construct_runtime_ui/undefined_inherent_part.rs:66:1 + | +66 | / construct_runtime! { +67 | | pub struct Runtime +68 | | { +69 | | System: frame_system expanded::{}::{Pallet, Call, Storage, Config, Event}, +70 | | Pallet: pallet expanded::{}::{Pallet, Inherent}, +71 | | } +72 | | } + | |_^ the trait `ProvideInherent` is not implemented for `pallet::Pallet` + | + = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/substrate/frame/support/test/tests/pallet.rs b/substrate/frame/support/test/tests/pallet.rs index 7bee4b7c8206..607f54ccc131 100644 --- a/substrate/frame/support/test/tests/pallet.rs +++ b/substrate/frame/support/test/tests/pallet.rs @@ -209,7 +209,7 @@ pub mod pallet { where T::AccountId: From + From + SomeAssociation1, { - /// Doc comment put in metadata + /// call foo doc comment put in metadata #[pallet::call_index(0)] #[pallet::weight(Weight::from_parts(*foo as u64, 0))] pub fn foo( @@ -225,7 +225,7 @@ pub mod pallet { Ok(().into()) } - /// Doc comment put in metadata + /// call foo_storage_layer doc comment put in metadata #[pallet::call_index(1)] #[pallet::weight({1})] pub fn foo_storage_layer( @@ -270,7 +270,7 @@ pub mod pallet { #[pallet::error] #[derive(PartialEq, Eq)] pub enum Error { - /// doc comment put into metadata + /// error doc comment put in metadata InsufficientProposersBalance, NonExistentStorageValue, Code(u8), @@ -287,9 +287,8 @@ pub mod pallet { where T::AccountId: SomeAssociation1 + From, { - /// doc comment put in metadata + /// event doc comment put in metadata Proposed(::AccountId), - /// doc Spending(BalanceOf), Something(u32), SomethingElse(::_1), @@ -750,8 +749,7 @@ pub type UncheckedExtrinsic = sp_runtime::testing::TestXt>; frame_support::construct_runtime!( - pub struct Runtime - { + pub struct Runtime { // Exclude part `Storage` in order not to check its metadata in tests. System: frame_system exclude_parts { Pallet, Storage }, Example: pallet, @@ -772,6 +770,14 @@ fn _ensure_call_is_correctly_excluded_and_included(call: RuntimeCall) { } } +fn maybe_docs(doc: Vec<&'static str>) -> Vec<&'static str> { + if cfg!(feature = "no-metadata-docs") { + vec![] + } else { + doc + } +} + #[test] fn transactional_works() { TestExternalities::default().execute_with(|| { @@ -1362,19 +1368,47 @@ fn migrate_from_pallet_version_to_storage_version() { }); } +#[test] +fn pallet_item_docs_in_metadata() { + // call + let call_variants = match meta_type::>().type_info().type_def { + scale_info::TypeDef::Variant(variants) => variants.variants, + _ => unreachable!(), + }; + + assert_eq!(call_variants[0].docs, maybe_docs(vec!["call foo doc comment put in metadata"])); + assert_eq!( + call_variants[1].docs, + maybe_docs(vec!["call foo_storage_layer doc comment put in metadata"]) + ); + assert!(call_variants[2].docs.is_empty()); + + // event + let event_variants = match meta_type::>().type_info().type_def { + scale_info::TypeDef::Variant(variants) => variants.variants, + _ => unreachable!(), + }; + + assert_eq!(event_variants[0].docs, maybe_docs(vec!["event doc comment put in metadata"])); + assert!(event_variants[1].docs.is_empty()); + + // error + let error_variants = match meta_type::>().type_info().type_def { + scale_info::TypeDef::Variant(variants) => variants.variants, + _ => unreachable!(), + }; + + assert_eq!(error_variants[0].docs, maybe_docs(vec!["error doc comment put in metadata"])); + assert!(error_variants[1].docs.is_empty()); + + // storage is already covered in the main `fn metadata` test. +} + #[test] fn metadata() { use codec::Decode; use frame_metadata::{v15::*, *}; - fn maybe_docs(doc: Vec<&'static str>) -> Vec<&'static str> { - if cfg!(feature = "no-metadata-docs") { - vec![] - } else { - doc - } - } - let readme = "Support code for the runtime.\n\nLicense: Apache-2.0\n"; let expected_pallet_doc = vec![" Pallet documentation", readme, readme]; diff --git a/substrate/frame/support/test/tests/pallet_outer_enums_explicit.rs b/substrate/frame/support/test/tests/pallet_outer_enums_explicit.rs index 79e9d6786717..33b96dea9486 100644 --- a/substrate/frame/support/test/tests/pallet_outer_enums_explicit.rs +++ b/substrate/frame/support/test/tests/pallet_outer_enums_explicit.rs @@ -90,6 +90,7 @@ fn module_error_outer_enum_expand_explicit() { frame_system::Error::NonDefaultComposite => (), frame_system::Error::NonZeroRefCount => (), frame_system::Error::CallFiltered => (), + frame_system::Error::MultiBlockMigrationsOngoing => (), #[cfg(feature = "experimental")] frame_system::Error::InvalidTask => (), #[cfg(feature = "experimental")] diff --git a/substrate/frame/support/test/tests/pallet_outer_enums_implicit.rs b/substrate/frame/support/test/tests/pallet_outer_enums_implicit.rs index 4bd8ee0bb39a..db006fe79359 100644 --- a/substrate/frame/support/test/tests/pallet_outer_enums_implicit.rs +++ b/substrate/frame/support/test/tests/pallet_outer_enums_implicit.rs @@ -90,6 +90,7 @@ fn module_error_outer_enum_expand_implicit() { frame_system::Error::NonDefaultComposite => (), frame_system::Error::NonZeroRefCount => (), frame_system::Error::CallFiltered => (), + frame_system::Error::MultiBlockMigrationsOngoing => (), #[cfg(feature = "experimental")] frame_system::Error::InvalidTask => (), #[cfg(feature = "experimental")] diff --git a/substrate/frame/support/test/tests/runtime_metadata.rs b/substrate/frame/support/test/tests/runtime_metadata.rs index bb7f7d2822e7..40e70b219ba9 100644 --- a/substrate/frame/support/test/tests/runtime_metadata.rs +++ b/substrate/frame/support/test/tests/runtime_metadata.rs @@ -101,7 +101,7 @@ sp_api::impl_runtime_apis! { fn execute_block(_: Block) { unimplemented!() } - fn initialize_block(_: &::Header) { + fn initialize_block(_: &::Header) -> sp_runtime::ExtrinsicInclusionMode { unimplemented!() } } @@ -200,8 +200,8 @@ fn runtime_metadata() { name: "header", ty: meta_type::<&::Header>(), }], - output: meta_type::<()>(), - docs: maybe_docs(vec![" Initialize a block with the given header."]), + output: meta_type::(), + docs: maybe_docs(vec![" Initialize a block with the given header and return the runtime executive mode."]), }, ], docs: maybe_docs(vec![ diff --git a/substrate/frame/system/src/lib.rs b/substrate/frame/system/src/lib.rs index 01df09106d99..3b659916379e 100644 --- a/substrate/frame/system/src/lib.rs +++ b/substrate/frame/system/src/lib.rs @@ -130,11 +130,13 @@ use frame_support::{ DispatchResult, DispatchResultWithPostInfo, PerDispatchClass, PostDispatchInfo, }, ensure, impl_ensure_origin_with_arg_ignoring_arg, + migrations::MultiStepMigrator, pallet_prelude::Pays, storage::{self, StorageStreamIter}, traits::{ ConstU32, Contains, EnsureOrigin, EnsureOriginWithArg, Get, HandleLifetime, - OnKilledAccount, OnNewAccount, OriginTrait, PalletInfo, SortedMembers, StoredMap, TypedGet, + OnKilledAccount, OnNewAccount, OnRuntimeUpgrade, OriginTrait, PalletInfo, SortedMembers, + StoredMap, TypedGet, }, Parameter, }; @@ -169,6 +171,7 @@ pub use extensions::{ // Backward compatible re-export. pub use extensions::check_mortality::CheckMortality as CheckEra; pub use frame_support::dispatch::RawOrigin; +use frame_support::traits::{PostInherents, PostTransactions, PreInherents}; pub use weights::WeightInfo; const LOG_TARGET: &str = "runtime::system"; @@ -299,6 +302,11 @@ pub mod pallet { type BaseCallFilter = frame_support::traits::Everything; type BlockHashCount = frame_support::traits::ConstU64<10>; type OnSetCode = (); + type SingleBlockMigrations = (); + type MultiBlockMigrator = (); + type PreInherents = (); + type PostInherents = (); + type PostTransactions = (); } /// Default configurations of this pallet in a solo-chain environment. @@ -393,6 +401,11 @@ pub mod pallet { /// The set code logic, just the default since we're not a parachain. type OnSetCode = (); + type SingleBlockMigrations = (); + type MultiBlockMigrator = (); + type PreInherents = (); + type PostInherents = (); + type PostTransactions = (); } /// Default configurations of this pallet in a relay-chain environment. @@ -572,6 +585,35 @@ pub mod pallet { /// The maximum number of consumers allowed on a single account. type MaxConsumers: ConsumerLimits; + + /// All migrations that should run in the next runtime upgrade. + /// + /// These used to be formerly configured in `Executive`. Parachains need to ensure that + /// running all these migrations in one block will not overflow the weight limit of a block. + /// The migrations are run *before* the pallet `on_runtime_upgrade` hooks, just like the + /// `OnRuntimeUpgrade` migrations. + type SingleBlockMigrations: OnRuntimeUpgrade; + + /// The migrator that is used to run Multi-Block-Migrations. + /// + /// Can be set to [`pallet-migrations`] or an alternative implementation of the interface. + /// The diagram in `frame_executive::block_flowchart` explains when it runs. + type MultiBlockMigrator: MultiStepMigrator; + + /// A callback that executes in *every block* directly before all inherents were applied. + /// + /// See `frame_executive::block_flowchart` for a in-depth explanation when it runs. + type PreInherents: PreInherents; + + /// A callback that executes in *every block* directly after all inherents were applied. + /// + /// See `frame_executive::block_flowchart` for a in-depth explanation when it runs. + type PostInherents: PostInherents; + + /// A callback that executes in *every block* directly after all transactions were applied. + /// + /// See `frame_executive::block_flowchart` for a in-depth explanation when it runs. + type PostTransactions: PostTransactions; } #[pallet::pallet] @@ -619,6 +661,9 @@ pub mod pallet { } /// Set the new runtime code without doing any checks of the given `code`. + /// + /// Note that runtime upgrades will not run if this is called with a not-increasing spec + /// version! #[pallet::call_index(3)] #[pallet::weight((T::SystemWeightInfo::set_code(), DispatchClass::Operational))] pub fn set_code_without_checks( @@ -814,6 +859,8 @@ pub mod pallet { NonZeroRefCount, /// The origin filter prevent the call to be dispatched. CallFiltered, + /// A multi-block migration is ongoing and prevents the current code from being replaced. + MultiBlockMigrationsOngoing, #[cfg(feature = "experimental")] /// The specified [`Task`] is not valid. InvalidTask, @@ -845,6 +892,10 @@ pub mod pallet { #[pallet::storage] pub(super) type ExtrinsicCount = StorageValue<_, u32>; + /// Whether all inherents have been applied. + #[pallet::storage] + pub type InherentsApplied = StorageValue<_, bool, ValueQuery>; + /// The current weight for the block. #[pallet::storage] #[pallet::whitelist_storage] @@ -1373,6 +1424,19 @@ impl Pallet { Self::deposit_event(Event::CodeUpdated); } + /// Whether all inherents have been applied. + pub fn inherents_applied() -> bool { + InherentsApplied::::get() + } + + /// Note that all inherents have been applied. + /// + /// Should be called immediately after all inherents have been applied. Must be called at least + /// once per block. + pub fn note_inherents_applied() { + InherentsApplied::::put(true); + } + /// Increment the reference counter on an account. #[deprecated = "Use `inc_consumers` instead"] pub fn inc_ref(who: &T::AccountId) { @@ -1692,6 +1756,7 @@ impl Pallet { >::put(digest); >::put(parent_hash); >::insert(*number - One::one(), parent_hash); + >::kill(); // Remove previous block data from storage BlockWeight::::kill(); @@ -1738,6 +1803,7 @@ impl Pallet { ExecutionPhase::::kill(); AllExtrinsicsLen::::kill(); storage::unhashed::kill(well_known_keys::INTRABLOCK_ENTROPY); + InherentsApplied::::kill(); // The following fields // @@ -1981,10 +2047,14 @@ impl Pallet { /// Determine whether or not it is possible to update the code. /// - /// Checks the given code if it is a valid runtime wasm blob by instantianting + /// Checks the given code if it is a valid runtime wasm blob by instantiating /// it and extracting the runtime version of it. It checks that the runtime version /// of the old and new runtime has the same spec name and that the spec version is increasing. pub fn can_set_code(code: &[u8]) -> Result<(), sp_runtime::DispatchError> { + if T::MultiBlockMigrator::ongoing() { + return Err(Error::::MultiBlockMigrationsOngoing.into()) + } + let current_version = T::Version::get(); let new_version = sp_io::misc::runtime_version(code) .and_then(|v| RuntimeVersion::decode(&mut &v[..]).ok()) diff --git a/substrate/frame/system/src/mock.rs b/substrate/frame/system/src/mock.rs index c4108099e39f..7059845a7df3 100644 --- a/substrate/frame/system/src/mock.rs +++ b/substrate/frame/system/src/mock.rs @@ -86,6 +86,22 @@ impl Config for Test { type Version = Version; type AccountData = u32; type OnKilledAccount = RecordKilled; + type MultiBlockMigrator = MockedMigrator; +} + +parameter_types! { + pub static Ongoing: bool = false; +} + +pub struct MockedMigrator; +impl frame_support::migrations::MultiStepMigrator for MockedMigrator { + fn ongoing() -> bool { + Ongoing::get() + } + + fn step() -> Weight { + Weight::zero() + } } pub type SysEvent = frame_system::Event; diff --git a/substrate/frame/system/src/tests.rs b/substrate/frame/system/src/tests.rs index e437e7f9f39b..b889b5ca046e 100644 --- a/substrate/frame/system/src/tests.rs +++ b/substrate/frame/system/src/tests.rs @@ -675,6 +675,28 @@ fn set_code_with_real_wasm_blob() { }); } +#[test] +fn set_code_rejects_during_mbm() { + Ongoing::set(true); + + let executor = substrate_test_runtime_client::new_native_or_wasm_executor(); + let mut ext = new_test_ext(); + ext.register_extension(sp_core::traits::ReadRuntimeVersionExt::new(executor)); + ext.execute_with(|| { + System::set_block_number(1); + let res = System::set_code( + RawOrigin::Root.into(), + substrate_test_runtime_client::runtime::wasm_binary_unwrap().to_vec(), + ); + assert_eq!( + res, + Err(DispatchErrorWithPostInfo::from(Error::::MultiBlockMigrationsOngoing)) + ); + + assert!(System::events().is_empty()); + }); +} + #[test] fn set_code_via_authorization_works() { let executor = substrate_test_runtime_client::new_native_or_wasm_executor(); diff --git a/substrate/primitives/api/proc-macro/src/decl_runtime_apis.rs b/substrate/primitives/api/proc-macro/src/decl_runtime_apis.rs index 2b1e65ec8852..e34e4c0e7672 100644 --- a/substrate/primitives/api/proc-macro/src/decl_runtime_apis.rs +++ b/substrate/primitives/api/proc-macro/src/decl_runtime_apis.rs @@ -456,6 +456,7 @@ impl<'a> ToClientSideDecl<'a> { |err| #crate_::ApiError::FailedToDecodeReturnValue { function: #function_name, error: err, + raw: r.clone(), } ) ) diff --git a/substrate/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs b/substrate/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs index c1339ff6621b..1761e0ac9dbf 100644 --- a/substrate/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs +++ b/substrate/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs @@ -158,7 +158,7 @@ fn implement_common_api_traits(block_type: TypePath, self_ty: Type) -> Result::Hash, _: &<#block_type as #crate_::BlockT>::Header, - ) -> std::result::Result<(), #crate_::ApiError> { + ) -> std::result::Result<#crate_::__private::ExtrinsicInclusionMode, #crate_::ApiError> { unimplemented!("`Core::initialize_block` not implemented for runtime api mocks") } } diff --git a/substrate/primitives/api/src/lib.rs b/substrate/primitives/api/src/lib.rs index 190de1ab3fde..a945b9f21f3c 100644 --- a/substrate/primitives/api/src/lib.rs +++ b/substrate/primitives/api/src/lib.rs @@ -101,7 +101,7 @@ pub mod __private { generic::BlockId, traits::{Block as BlockT, Hash as HashT, HashingFor, Header as HeaderT, NumberFor}, transaction_validity::TransactionValidity, - RuntimeString, TransactionOutcome, + ExtrinsicInclusionMode, RuntimeString, TransactionOutcome, }; pub use sp_std::{mem, slice, vec}; pub use sp_version::{create_apis_vec, ApiId, ApisVec, RuntimeVersion}; @@ -115,11 +115,11 @@ pub use sp_core::traits::CallContext; use sp_core::OpaqueMetadata; #[cfg(feature = "std")] use sp_externalities::{Extension, Extensions}; -use sp_runtime::traits::Block as BlockT; #[cfg(feature = "std")] use sp_runtime::traits::HashingFor; #[cfg(feature = "std")] pub use sp_runtime::TransactionOutcome; +use sp_runtime::{traits::Block as BlockT, ExtrinsicInclusionMode}; #[cfg(feature = "std")] pub use sp_state_machine::StorageProof; #[cfg(feature = "std")] @@ -280,7 +280,7 @@ pub use sp_api_proc_macro::decl_runtime_apis; /// ```rust /// use sp_version::create_runtime_str; /// # -/// # use sp_runtime::traits::Block as BlockT; +/// # use sp_runtime::{ExtrinsicInclusionMode, traits::Block as BlockT}; /// # use sp_test_primitives::Block; /// # /// # /// The declaration of the `Runtime` type is done by the `construct_runtime!` macro @@ -307,7 +307,9 @@ pub use sp_api_proc_macro::decl_runtime_apis; /// # unimplemented!() /// # } /// # fn execute_block(_block: Block) {} -/// # fn initialize_block(_header: &::Header) {} +/// # fn initialize_block(_header: &::Header) -> ExtrinsicInclusionMode { +/// # unimplemented!() +/// # } /// # } /// /// impl self::Balance for Runtime { @@ -540,11 +542,12 @@ pub fn init_runtime_logger() { #[cfg(feature = "std")] #[derive(Debug, thiserror::Error)] pub enum ApiError { - #[error("Failed to decode return value of {function}")] + #[error("Failed to decode return value of {function}: {error} raw data: {raw:?}")] FailedToDecodeReturnValue { function: &'static str, #[source] error: codec::Error, + raw: Vec, }, #[error("Failed to convert return value from runtime to node of {function}")] FailedToConvertReturnValue { @@ -800,15 +803,18 @@ pub fn deserialize_runtime_api_info(bytes: [u8; RUNTIME_API_INFO_SIZE]) -> ([u8; decl_runtime_apis! { /// The `Core` runtime api that every Substrate runtime needs to implement. #[core_trait] - #[api_version(4)] + #[api_version(5)] pub trait Core { /// Returns the version of the runtime. fn version() -> RuntimeVersion; /// Execute the given block. fn execute_block(block: Block); /// Initialize a block with the given header. + #[changed_in(5)] #[renamed("initialise_block", 2)] fn initialize_block(header: &::Header); + /// Initialize a block with the given header and return the runtime executive mode. + fn initialize_block(header: &::Header) -> ExtrinsicInclusionMode; } /// The `Metadata` api trait that returns metadata for the runtime. diff --git a/substrate/primitives/api/test/tests/decl_and_impl.rs b/substrate/primitives/api/test/tests/decl_and_impl.rs index d68470551d20..211a08561fd4 100644 --- a/substrate/primitives/api/test/tests/decl_and_impl.rs +++ b/substrate/primitives/api/test/tests/decl_and_impl.rs @@ -139,7 +139,7 @@ impl_runtime_apis! { fn execute_block(_: Block) { unimplemented!() } - fn initialize_block(_: &::Header) { + fn initialize_block(_: &::Header) -> sp_runtime::ExtrinsicInclusionMode { unimplemented!() } } diff --git a/substrate/primitives/api/test/tests/ui/impl_incorrect_method_signature.rs b/substrate/primitives/api/test/tests/ui/impl_incorrect_method_signature.rs index 43718e4cd04a..262a874213a5 100644 --- a/substrate/primitives/api/test/tests/ui/impl_incorrect_method_signature.rs +++ b/substrate/primitives/api/test/tests/ui/impl_incorrect_method_signature.rs @@ -40,7 +40,7 @@ sp_api::impl_runtime_apis! { fn execute_block(_: Block) { unimplemented!() } - fn initialize_block(_: &::Header) { + fn initialize_block(_: &::Header) -> sp_runtime::ExtrinsicInclusionMode { unimplemented!() } } diff --git a/substrate/primitives/api/test/tests/ui/impl_missing_version.rs b/substrate/primitives/api/test/tests/ui/impl_missing_version.rs index 560257b5168c..58850ab343fb 100644 --- a/substrate/primitives/api/test/tests/ui/impl_missing_version.rs +++ b/substrate/primitives/api/test/tests/ui/impl_missing_version.rs @@ -45,7 +45,7 @@ sp_api::impl_runtime_apis! { fn execute_block(_: Block) { unimplemented!() } - fn initialize_block(_: &::Header) { + fn initialize_block(_: &::Header) -> sp_runtime::ExtrinsicInclusionMode { unimplemented!() } } diff --git a/substrate/primitives/api/test/tests/ui/missing_versioned_method.rs b/substrate/primitives/api/test/tests/ui/missing_versioned_method.rs index 6ead545f85a1..70f75d065154 100644 --- a/substrate/primitives/api/test/tests/ui/missing_versioned_method.rs +++ b/substrate/primitives/api/test/tests/ui/missing_versioned_method.rs @@ -44,7 +44,7 @@ sp_api::impl_runtime_apis! { fn execute_block(_: Block) { unimplemented!() } - fn initialize_block(_: &::Header) { + fn initialize_block(_: &::Header) -> sp_runtime::ExtrinsicInclusionMode { unimplemented!() } } diff --git a/substrate/primitives/api/test/tests/ui/missing_versioned_method_multiple_vers.rs b/substrate/primitives/api/test/tests/ui/missing_versioned_method_multiple_vers.rs index 8eebc1d79bab..63032000040b 100644 --- a/substrate/primitives/api/test/tests/ui/missing_versioned_method_multiple_vers.rs +++ b/substrate/primitives/api/test/tests/ui/missing_versioned_method_multiple_vers.rs @@ -47,7 +47,7 @@ sp_api::impl_runtime_apis! { fn execute_block(_: Block) { unimplemented!() } - fn initialize_block(_: &::Header) { + fn initialize_block(_: &::Header) -> sp_runtime::ExtrinsicInclusionMode { unimplemented!() } } diff --git a/substrate/primitives/api/test/tests/ui/positive_cases/custom_where_bound.rs b/substrate/primitives/api/test/tests/ui/positive_cases/custom_where_bound.rs index 594556d57be5..0858813bc999 100644 --- a/substrate/primitives/api/test/tests/ui/positive_cases/custom_where_bound.rs +++ b/substrate/primitives/api/test/tests/ui/positive_cases/custom_where_bound.rs @@ -51,7 +51,7 @@ sp_api::impl_runtime_apis! { fn execute_block(_: Block) { unimplemented!() } - fn initialize_block(_: &::Header) { + fn initialize_block(_: &::Header) -> sp_runtime::ExtrinsicInclusionMode { unimplemented!() } } diff --git a/substrate/primitives/api/test/tests/ui/positive_cases/default_impls.rs b/substrate/primitives/api/test/tests/ui/positive_cases/default_impls.rs index ae573238ffe1..3e0cb79156c8 100644 --- a/substrate/primitives/api/test/tests/ui/positive_cases/default_impls.rs +++ b/substrate/primitives/api/test/tests/ui/positive_cases/default_impls.rs @@ -46,7 +46,7 @@ sp_api::impl_runtime_apis! { fn execute_block(_: Block) { unimplemented!() } - fn initialize_block(_: &::Header) { + fn initialize_block(_: &::Header) -> sp_runtime::ExtrinsicInclusionMode { unimplemented!() } } diff --git a/substrate/primitives/api/test/tests/ui/type_reference_in_impl_runtime_apis_call.rs b/substrate/primitives/api/test/tests/ui/type_reference_in_impl_runtime_apis_call.rs index 921bf0d04351..b2caea7ab7e4 100644 --- a/substrate/primitives/api/test/tests/ui/type_reference_in_impl_runtime_apis_call.rs +++ b/substrate/primitives/api/test/tests/ui/type_reference_in_impl_runtime_apis_call.rs @@ -42,7 +42,7 @@ sp_api::impl_runtime_apis! { fn execute_block(_: Block) { unimplemented!() } - fn initialize_block(_: &::Header) { + fn initialize_block(_: &::Header) -> sp_runtime::ExtrinsicInclusionMode { unimplemented!() } } diff --git a/substrate/primitives/runtime/src/lib.rs b/substrate/primitives/runtime/src/lib.rs index ddf92554c830..44bf3c969e54 100644 --- a/substrate/primitives/runtime/src/lib.rs +++ b/substrate/primitives/runtime/src/lib.rs @@ -998,6 +998,16 @@ impl TransactionOutcome { } } +/// Confines the kind of extrinsics that can be included in a block. +#[derive(Debug, Default, PartialEq, Eq, Clone, Copy, Encode, Decode, TypeInfo)] +pub enum ExtrinsicInclusionMode { + /// All extrinsics are allowed to be included in this block. + #[default] + AllExtrinsics, + /// Inherents are allowed to be included. + OnlyInherents, +} + #[cfg(test)] mod tests { use crate::traits::BlakeTwo256; diff --git a/substrate/primitives/weights/src/lib.rs b/substrate/primitives/weights/src/lib.rs index aede9473535a..b2c956266e2c 100644 --- a/substrate/primitives/weights/src/lib.rs +++ b/substrate/primitives/weights/src/lib.rs @@ -18,9 +18,6 @@ //! # Primitives for transaction weighting. #![cfg_attr(not(feature = "std"), no_std)] -// TODO remove once `OldWeight` is gone. I dont know why this is needed, maybe by one of the macros -// of `OldWeight`. -#![allow(deprecated)] extern crate self as sp_weights; @@ -28,7 +25,7 @@ mod weight_meter; mod weight_v2; use bounded_collections::Get; -use codec::{CompactAs, Decode, Encode, MaxEncodedLen}; +use codec::{Decode, Encode}; use scale_info::TypeInfo; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; @@ -52,28 +49,6 @@ pub mod constants { pub const WEIGHT_PROOF_SIZE_PER_KB: u64 = 1024; } -/// The old weight type. -/// -/// NOTE: This type exists purely for compatibility purposes! Use [`weight_v2::Weight`] in all other -/// cases. -#[derive( - Decode, - Encode, - CompactAs, - PartialEq, - Eq, - Clone, - Copy, - RuntimeDebug, - Default, - MaxEncodedLen, - TypeInfo, -)] -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[cfg_attr(feature = "serde", serde(transparent))] -#[deprecated(note = "Will be removed soon; use `Weight` instead.")] -pub struct OldWeight(pub u64); - /// The weight of database operations that the runtime can invoke. /// /// NOTE: This is currently only measured in computational time, and will probably diff --git a/substrate/primitives/weights/src/weight_meter.rs b/substrate/primitives/weights/src/weight_meter.rs index 1738948e4c3c..cfe8396ae6d6 100644 --- a/substrate/primitives/weights/src/weight_meter.rs +++ b/substrate/primitives/weights/src/weight_meter.rs @@ -118,13 +118,6 @@ impl WeightMeter { debug_assert!(self.consumed.all_lte(self.limit), "Weight counter overflow"); } - /// Consume the given weight after checking that it can be consumed and return `true`. Otherwise - /// do nothing and return `false`. - #[deprecated(note = "Use `try_consume` instead. Will be removed after December 2023.")] - pub fn check_accrue(&mut self, w: Weight) -> bool { - self.try_consume(w).is_ok() - } - /// Consume the given weight after checking that it can be consumed. /// /// Returns `Ok` if the weight can be consumed or otherwise an `Err`. @@ -139,12 +132,6 @@ impl WeightMeter { }) } - /// Check if the given weight can be consumed. - #[deprecated(note = "Use `can_consume` instead. Will be removed after December 2023.")] - pub fn can_accrue(&self, w: Weight) -> bool { - self.can_consume(w) - } - /// Check if the given weight can be consumed. pub fn can_consume(&self, w: Weight) -> bool { self.consumed.checked_add(&w).map_or(false, |t| t.all_lte(self.limit)) @@ -165,80 +152,80 @@ mod tests { fn weight_meter_remaining_works() { let mut meter = WeightMeter::with_limit(Weight::from_parts(10, 20)); - assert!(meter.check_accrue(Weight::from_parts(5, 0))); + assert_eq!(meter.try_consume(Weight::from_parts(5, 0)), Ok(())); assert_eq!(meter.consumed, Weight::from_parts(5, 0)); assert_eq!(meter.remaining(), Weight::from_parts(5, 20)); - assert!(meter.check_accrue(Weight::from_parts(2, 10))); + assert_eq!(meter.try_consume(Weight::from_parts(2, 10)), Ok(())); assert_eq!(meter.consumed, Weight::from_parts(7, 10)); assert_eq!(meter.remaining(), Weight::from_parts(3, 10)); - assert!(meter.check_accrue(Weight::from_parts(3, 10))); + assert_eq!(meter.try_consume(Weight::from_parts(3, 10)), Ok(())); assert_eq!(meter.consumed, Weight::from_parts(10, 20)); assert_eq!(meter.remaining(), Weight::from_parts(0, 0)); } #[test] - fn weight_meter_can_accrue_works() { + fn weight_meter_can_consume_works() { let meter = WeightMeter::with_limit(Weight::from_parts(1, 1)); - assert!(meter.can_accrue(Weight::from_parts(0, 0))); - assert!(meter.can_accrue(Weight::from_parts(1, 1))); - assert!(!meter.can_accrue(Weight::from_parts(0, 2))); - assert!(!meter.can_accrue(Weight::from_parts(2, 0))); - assert!(!meter.can_accrue(Weight::from_parts(2, 2))); + assert!(meter.can_consume(Weight::from_parts(0, 0))); + assert!(meter.can_consume(Weight::from_parts(1, 1))); + assert!(!meter.can_consume(Weight::from_parts(0, 2))); + assert!(!meter.can_consume(Weight::from_parts(2, 0))); + assert!(!meter.can_consume(Weight::from_parts(2, 2))); } #[test] - fn weight_meter_check_accrue_works() { + fn weight_meter_try_consume_works() { let mut meter = WeightMeter::with_limit(Weight::from_parts(2, 2)); - assert!(meter.check_accrue(Weight::from_parts(0, 0))); - assert!(meter.check_accrue(Weight::from_parts(1, 1))); - assert!(!meter.check_accrue(Weight::from_parts(0, 2))); - assert!(!meter.check_accrue(Weight::from_parts(2, 0))); - assert!(!meter.check_accrue(Weight::from_parts(2, 2))); - assert!(meter.check_accrue(Weight::from_parts(0, 1))); - assert!(meter.check_accrue(Weight::from_parts(1, 0))); + assert_eq!(meter.try_consume(Weight::from_parts(0, 0)), Ok(())); + assert_eq!(meter.try_consume(Weight::from_parts(1, 1)), Ok(())); + assert_eq!(meter.try_consume(Weight::from_parts(0, 2)), Err(())); + assert_eq!(meter.try_consume(Weight::from_parts(2, 0)), Err(())); + assert_eq!(meter.try_consume(Weight::from_parts(2, 2)), Err(())); + assert_eq!(meter.try_consume(Weight::from_parts(0, 1)), Ok(())); + assert_eq!(meter.try_consume(Weight::from_parts(1, 0)), Ok(())); } #[test] - fn weight_meter_check_and_can_accrue_works() { + fn weight_meter_check_and_can_consume_works() { let mut meter = WeightMeter::new(); - assert!(meter.can_accrue(Weight::from_parts(u64::MAX, 0))); - assert!(meter.check_accrue(Weight::from_parts(u64::MAX, 0))); + assert!(meter.can_consume(Weight::from_parts(u64::MAX, 0))); + assert_eq!(meter.try_consume(Weight::from_parts(u64::MAX, 0)), Ok(())); - assert!(meter.can_accrue(Weight::from_parts(0, u64::MAX))); - assert!(meter.check_accrue(Weight::from_parts(0, u64::MAX))); + assert!(meter.can_consume(Weight::from_parts(0, u64::MAX))); + assert_eq!(meter.try_consume(Weight::from_parts(0, u64::MAX)), Ok(())); - assert!(!meter.can_accrue(Weight::from_parts(0, 1))); - assert!(!meter.check_accrue(Weight::from_parts(0, 1))); + assert!(!meter.can_consume(Weight::from_parts(0, 1))); + assert_eq!(meter.try_consume(Weight::from_parts(0, 1)), Err(())); - assert!(!meter.can_accrue(Weight::from_parts(1, 0))); - assert!(!meter.check_accrue(Weight::from_parts(1, 0))); + assert!(!meter.can_consume(Weight::from_parts(1, 0))); + assert_eq!(meter.try_consume(Weight::from_parts(1, 0)), Err(())); - assert!(meter.can_accrue(Weight::zero())); - assert!(meter.check_accrue(Weight::zero())); + assert!(meter.can_consume(Weight::zero())); + assert_eq!(meter.try_consume(Weight::zero()), Ok(())); } #[test] fn consumed_ratio_works() { let mut meter = WeightMeter::with_limit(Weight::from_parts(10, 20)); - assert!(meter.check_accrue(Weight::from_parts(5, 0))); + assert_eq!(meter.try_consume(Weight::from_parts(5, 0)), Ok(())); assert_eq!(meter.consumed_ratio(), Perbill::from_percent(50)); - assert!(meter.check_accrue(Weight::from_parts(0, 12))); + assert_eq!(meter.try_consume(Weight::from_parts(0, 12)), Ok(())); assert_eq!(meter.consumed_ratio(), Perbill::from_percent(60)); - assert!(meter.check_accrue(Weight::from_parts(2, 0))); + assert_eq!(meter.try_consume(Weight::from_parts(2, 0)), Ok(())); assert_eq!(meter.consumed_ratio(), Perbill::from_percent(70)); - assert!(meter.check_accrue(Weight::from_parts(0, 4))); + assert_eq!(meter.try_consume(Weight::from_parts(0, 4)), Ok(())); assert_eq!(meter.consumed_ratio(), Perbill::from_percent(80)); - assert!(meter.check_accrue(Weight::from_parts(3, 0))); + assert_eq!(meter.try_consume(Weight::from_parts(3, 0)), Ok(())); assert_eq!(meter.consumed_ratio(), Perbill::from_percent(100)); - assert!(meter.check_accrue(Weight::from_parts(0, 4))); + assert_eq!(meter.try_consume(Weight::from_parts(0, 4)), Ok(())); assert_eq!(meter.consumed_ratio(), Perbill::from_percent(100)); } diff --git a/substrate/test-utils/runtime/src/lib.rs b/substrate/test-utils/runtime/src/lib.rs index db9ff187b707..63e0aa6e1379 100644 --- a/substrate/test-utils/runtime/src/lib.rs +++ b/substrate/test-utils/runtime/src/lib.rs @@ -61,7 +61,7 @@ use sp_runtime::{ create_runtime_str, impl_opaque_keys, traits::{BlakeTwo256, Block as BlockT, DispatchInfoOf, NumberFor, Verify}, transaction_validity::{TransactionSource, TransactionValidity, TransactionValidityError}, - ApplyExtrinsicResult, Perbill, + ApplyExtrinsicResult, ExtrinsicInclusionMode, Perbill, }; #[cfg(any(feature = "std", test))] use sp_version::NativeVersion; @@ -480,9 +480,9 @@ impl_runtime_apis! { Executive::execute_block(block); } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> ExtrinsicInclusionMode { log::trace!(target: LOG_TARGET, "initialize_block: {header:#?}"); - Executive::initialize_block(header); + Executive::initialize_block(header) } } diff --git a/substrate/test-utils/runtime/transaction-pool/src/lib.rs b/substrate/test-utils/runtime/transaction-pool/src/lib.rs index 8c8345b06bd3..5202e6e65154 100644 --- a/substrate/test-utils/runtime/transaction-pool/src/lib.rs +++ b/substrate/test-utils/runtime/transaction-pool/src/lib.rs @@ -81,6 +81,7 @@ pub struct ChainState { pub block_by_hash: HashMap, pub nonces: HashMap, pub invalid_hashes: HashSet, + pub priorities: HashMap, } /// Test Api for transaction pool. @@ -214,6 +215,22 @@ impl TestApi { self.chain.write().invalid_hashes.insert(Self::hash_and_length_inner(xts).0); } + /// Remove a transaction that was previously declared as invalid via `[Self::add_invalid]`. + /// + /// Next time transaction pool will try to validate this + /// extrinsic, api will succeed. + pub fn remove_invalid(&self, xts: &Extrinsic) { + self.chain.write().invalid_hashes.remove(&Self::hash_and_length_inner(xts).0); + } + + /// Set a transaction priority. + pub fn set_priority(&self, xts: &Extrinsic, priority: u64) { + self.chain + .write() + .priorities + .insert(Self::hash_and_length_inner(xts).0, priority); + } + /// Query validation requests received. pub fn validation_requests(&self) -> Vec { self.validation_requests.read().clone() @@ -300,8 +317,14 @@ impl ChainApi for TestApi { return ready(Ok(Err(TransactionValidityError::Invalid(InvalidTransaction::Custom(0))))) } - let mut validity = - ValidTransaction { priority: 1, requires, provides, longevity: 64, propagate: true }; + let priority = self.chain.read().priorities.get(&self.hash_and_length(&uxt).0).cloned(); + let mut validity = ValidTransaction { + priority: priority.unwrap_or(1), + requires, + provides, + longevity: 64, + propagate: true, + }; (self.valid_modifier.read())(&mut validity);