From bf3b284b0c3c4951b7f655c65f0c5d79e7a7da43 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 14 Sep 2023 10:20:36 -0500 Subject: [PATCH 1/5] GH-1523 GH-1631 Transition from dpos to hotstuff Producer schedule changes under hotstuff in next,next producer round. --- libraries/chain/block_header_state.cpp | 199 ++++++++++-------- libraries/chain/block_state.cpp | 3 +- libraries/chain/controller.cpp | 46 ++-- libraries/chain/fork_database.cpp | 5 + .../include/eosio/chain/block_header.hpp | 6 + .../eosio/chain/block_header_state.hpp | 34 ++- .../chain/include/eosio/chain/block_state.hpp | 1 + .../chain/include/eosio/chain/controller.hpp | 4 + libraries/hotstuff/chain_pacemaker.cpp | 4 + plugins/producer_plugin/producer_plugin.cpp | 17 +- 10 files changed, 207 insertions(+), 112 deletions(-) diff --git a/libraries/chain/block_header_state.cpp b/libraries/chain/block_header_state.cpp index 65fa92be3f..1d55254d6e 100644 --- a/libraries/chain/block_header_state.cpp +++ b/libraries/chain/block_header_state.cpp @@ -15,6 +15,12 @@ namespace eosio { namespace chain { const auto& protocol_features = pfa->protocol_features; return digest && protocol_features.find(*digest) != protocol_features.end(); } + + uint32_t get_next_next_round_block_num( block_timestamp_type t, uint32_t block_num ) { + auto index = t.slot % config::producer_repetitions; + // remainder of current + next round + return block_num + config::producer_repetitions - index + config::producer_repetitions; + } } producer_authority block_header_state::get_scheduled_producer( block_timestamp_type t )const { @@ -38,6 +44,7 @@ namespace eosio { namespace chain { } pending_block_header_state block_header_state::next( block_timestamp_type when, + bool hotstuff_activated, uint16_t num_prev_blocks_to_confirm )const { pending_block_header_state result; @@ -50,18 +57,9 @@ namespace eosio { namespace chain { auto proauth = get_scheduled_producer(when); - auto itr = producer_to_last_produced.find( proauth.producer_name ); - if( itr != producer_to_last_produced.end() ) { - EOS_ASSERT( itr->second < (block_num+1) - num_prev_blocks_to_confirm, producer_double_confirm, - "producer ${prod} double-confirming known range", - ("prod", proauth.producer_name)("num", block_num+1) - ("confirmed", num_prev_blocks_to_confirm)("last_produced", itr->second) ); - } - result.block_num = block_num + 1; result.previous = id; result.timestamp = when; - result.confirmed = num_prev_blocks_to_confirm; result.active_schedule_version = active_schedule.version; result.prev_activated_protocol_features = activated_protocol_features; @@ -72,103 +70,129 @@ namespace eosio { namespace chain { result.blockroot_merkle = blockroot_merkle; result.blockroot_merkle.append( id ); - /// grow the confirmed count - static_assert(std::numeric_limits::max() >= (config::max_producers * 2 / 3) + 1, "8bit confirmations may not be able to hold all of the needed confirmations"); - - // This uses the previous block active_schedule because thats the "schedule" that signs and therefore confirms _this_ block - auto num_active_producers = active_schedule.producers.size(); - uint32_t required_confs = (uint32_t)(num_active_producers * 2 / 3) + 1; + result.prev_pending_schedule = pending_schedule; + + if (hotstuff_activated) { + result.confirmed = hs_block_confirmed; + result.dpos_proposed_irreversible_blocknum = 0; + // fork_database will prefer hotstuff blocks over dpos blocks + result.dpos_irreversible_blocknum = hs_dpos_irreversible_blocknum; + // Change to active on the next().next() producer block_num + if( pending_schedule.schedule.producers.size() && + block_num >= detail::get_next_next_round_block_num(when, pending_schedule.schedule_lib_num)) { + result.active_schedule = pending_schedule.schedule; + result.was_pending_promoted = true; + } else { + result.active_schedule = active_schedule; + } - if( confirm_count.size() < config::maximum_tracked_dpos_confirmations ) { - result.confirm_count.reserve( confirm_count.size() + 1 ); - result.confirm_count = confirm_count; - result.confirm_count.resize( confirm_count.size() + 1 ); - result.confirm_count.back() = (uint8_t)required_confs; } else { - result.confirm_count.resize( confirm_count.size() ); - memcpy( &result.confirm_count[0], &confirm_count[1], confirm_count.size() - 1 ); - result.confirm_count.back() = (uint8_t)required_confs; - } + auto itr = producer_to_last_produced.find( proauth.producer_name ); + if( itr != producer_to_last_produced.end() ) { + EOS_ASSERT( itr->second < (block_num+1) - num_prev_blocks_to_confirm, producer_double_confirm, + "producer ${prod} double-confirming known range", + ("prod", proauth.producer_name)("num", block_num+1) + ("confirmed", num_prev_blocks_to_confirm)("last_produced", itr->second) ); + } - auto new_dpos_proposed_irreversible_blocknum = dpos_proposed_irreversible_blocknum; - - int32_t i = (int32_t)(result.confirm_count.size() - 1); - uint32_t blocks_to_confirm = num_prev_blocks_to_confirm + 1; /// confirm the head block too - while( i >= 0 && blocks_to_confirm ) { - --result.confirm_count[i]; - //idump((confirm_count[i])); - if( result.confirm_count[i] == 0 ) - { - uint32_t block_num_for_i = result.block_num - (uint32_t)(result.confirm_count.size() - 1 - i); - new_dpos_proposed_irreversible_blocknum = block_num_for_i; - //idump((dpos2_lib)(block_num)(dpos_irreversible_blocknum)); - - if (i == static_cast(result.confirm_count.size() - 1)) { - result.confirm_count.resize(0); - } else { - memmove( &result.confirm_count[0], &result.confirm_count[i + 1], result.confirm_count.size() - i - 1); - result.confirm_count.resize( result.confirm_count.size() - i - 1 ); - } + result.confirmed = num_prev_blocks_to_confirm; - break; - } - --i; - --blocks_to_confirm; - } + /// grow the confirmed count + static_assert(std::numeric_limits::max() >= (config::max_producers * 2 / 3) + 1, "8bit confirmations may not be able to hold all of the needed confirmations"); + + // This uses the previous block active_schedule because thats the "schedule" that signs and therefore confirms _this_ block + auto num_active_producers = active_schedule.producers.size(); + uint32_t required_confs = (uint32_t)(num_active_producers * 2 / 3) + 1; + + if( confirm_count.size() < config::maximum_tracked_dpos_confirmations ) { + result.confirm_count.reserve( confirm_count.size() + 1 ); + result.confirm_count = confirm_count; + result.confirm_count.resize( confirm_count.size() + 1 ); + result.confirm_count.back() = (uint8_t)required_confs; + } else { + result.confirm_count.resize( confirm_count.size() ); + memcpy( &result.confirm_count[0], &confirm_count[1], confirm_count.size() - 1 ); + result.confirm_count.back() = (uint8_t)required_confs; + } - result.dpos_proposed_irreversible_blocknum = new_dpos_proposed_irreversible_blocknum; - result.dpos_irreversible_blocknum = calc_dpos_last_irreversible( proauth.producer_name ); + auto new_dpos_proposed_irreversible_blocknum = dpos_proposed_irreversible_blocknum; + + int32_t i = (int32_t)(result.confirm_count.size() - 1); + uint32_t blocks_to_confirm = num_prev_blocks_to_confirm + 1; /// confirm the head block too + while( i >= 0 && blocks_to_confirm ) { + --result.confirm_count[i]; + //idump((confirm_count[i])); + if( result.confirm_count[i] == 0 ) + { + uint32_t block_num_for_i = result.block_num - (uint32_t)(result.confirm_count.size() - 1 - i); + new_dpos_proposed_irreversible_blocknum = block_num_for_i; + //idump((dpos2_lib)(block_num)(dpos_irreversible_blocknum)); + + if (i == static_cast(result.confirm_count.size() - 1)) { + result.confirm_count.resize(0); + } else { + memmove( &result.confirm_count[0], &result.confirm_count[i + 1], result.confirm_count.size() - i - 1); + result.confirm_count.resize( result.confirm_count.size() - i - 1 ); + } - result.prev_pending_schedule = pending_schedule; + break; + } + --i; + --blocks_to_confirm; + } - if( pending_schedule.schedule.producers.size() && - result.dpos_irreversible_blocknum >= pending_schedule.schedule_lib_num ) - { - result.active_schedule = pending_schedule.schedule; + result.dpos_proposed_irreversible_blocknum = new_dpos_proposed_irreversible_blocknum; + result.dpos_irreversible_blocknum = calc_dpos_last_irreversible( proauth.producer_name ); + + if( pending_schedule.schedule.producers.size() && + result.dpos_irreversible_blocknum >= pending_schedule.schedule_lib_num ) + { + result.active_schedule = pending_schedule.schedule; - flat_map new_producer_to_last_produced; + flat_map new_producer_to_last_produced; - for( const auto& pro : result.active_schedule.producers ) { - if( pro.producer_name == proauth.producer_name ) { - new_producer_to_last_produced[pro.producer_name] = result.block_num; - } else { - auto existing = producer_to_last_produced.find( pro.producer_name ); - if( existing != producer_to_last_produced.end() ) { - new_producer_to_last_produced[pro.producer_name] = existing->second; + for( const auto& pro : result.active_schedule.producers ) { + if( pro.producer_name == proauth.producer_name ) { + new_producer_to_last_produced[pro.producer_name] = result.block_num; } else { - new_producer_to_last_produced[pro.producer_name] = result.dpos_irreversible_blocknum; + auto existing = producer_to_last_produced.find( pro.producer_name ); + if( existing != producer_to_last_produced.end() ) { + new_producer_to_last_produced[pro.producer_name] = existing->second; + } else { + new_producer_to_last_produced[pro.producer_name] = result.dpos_irreversible_blocknum; + } } } - } - new_producer_to_last_produced[proauth.producer_name] = result.block_num; + new_producer_to_last_produced[proauth.producer_name] = result.block_num; - result.producer_to_last_produced = std::move( new_producer_to_last_produced ); + result.producer_to_last_produced = std::move( new_producer_to_last_produced ); - flat_map new_producer_to_last_implied_irb; + flat_map new_producer_to_last_implied_irb; - for( const auto& pro : result.active_schedule.producers ) { - if( pro.producer_name == proauth.producer_name ) { - new_producer_to_last_implied_irb[pro.producer_name] = dpos_proposed_irreversible_blocknum; - } else { - auto existing = producer_to_last_implied_irb.find( pro.producer_name ); - if( existing != producer_to_last_implied_irb.end() ) { - new_producer_to_last_implied_irb[pro.producer_name] = existing->second; + for( const auto& pro : result.active_schedule.producers ) { + if( pro.producer_name == proauth.producer_name ) { + new_producer_to_last_implied_irb[pro.producer_name] = dpos_proposed_irreversible_blocknum; } else { - new_producer_to_last_implied_irb[pro.producer_name] = result.dpos_irreversible_blocknum; + auto existing = producer_to_last_implied_irb.find( pro.producer_name ); + if( existing != producer_to_last_implied_irb.end() ) { + new_producer_to_last_implied_irb[pro.producer_name] = existing->second; + } else { + new_producer_to_last_implied_irb[pro.producer_name] = result.dpos_irreversible_blocknum; + } } } - } - result.producer_to_last_implied_irb = std::move( new_producer_to_last_implied_irb ); + result.producer_to_last_implied_irb = std::move( new_producer_to_last_implied_irb ); - result.was_pending_promoted = true; - } else { - result.active_schedule = active_schedule; - result.producer_to_last_produced = producer_to_last_produced; - result.producer_to_last_produced[proauth.producer_name] = result.block_num; - result.producer_to_last_implied_irb = producer_to_last_implied_irb; - result.producer_to_last_implied_irb[proauth.producer_name] = dpos_proposed_irreversible_blocknum; - } + result.was_pending_promoted = true; + } else { + result.active_schedule = active_schedule; + result.producer_to_last_produced = producer_to_last_produced; + result.producer_to_last_produced[proauth.producer_name] = result.block_num; + result.producer_to_last_implied_irb = producer_to_last_implied_irb; + result.producer_to_last_implied_irb[proauth.producer_name] = dpos_proposed_irreversible_blocknum; + } + } // !hotstuff_activated return result; } @@ -385,12 +409,13 @@ namespace eosio { namespace chain { const signed_block_header& h, vector&& _additional_signatures, const protocol_feature_set& pfs, + bool hotstuff_activated, const std::function&, const vector& )>& validator, bool skip_validate_signee )const { - return next( h.timestamp, h.confirmed ).finish_next( h, std::move(_additional_signatures), pfs, validator, skip_validate_signee ); + return next( h.timestamp, hotstuff_activated, h.confirmed ).finish_next( h, std::move(_additional_signatures), pfs, validator, skip_validate_signee ); } digest_type block_header_state::sig_digest()const { diff --git a/libraries/chain/block_state.cpp b/libraries/chain/block_state.cpp index fd614a6118..c1dbdf122c 100644 --- a/libraries/chain/block_state.cpp +++ b/libraries/chain/block_state.cpp @@ -77,12 +77,13 @@ namespace eosio { namespace chain { block_state::block_state( const block_header_state& prev, signed_block_ptr b, const protocol_feature_set& pfs, + bool hotstuff_activated, const std::function&, const vector& )>& validator, bool skip_validate_signee ) - :block_header_state( prev.next( *b, extract_additional_signatures(b, pfs, prev.activated_protocol_features), pfs, validator, skip_validate_signee ) ) + :block_header_state( prev.next( *b, extract_additional_signatures(b, pfs, prev.activated_protocol_features), pfs, hotstuff_activated, validator, skip_validate_signee ) ) ,block( std::move(b) ) {} diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index e4c6f89bb0..2e0d282d36 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -116,9 +116,10 @@ class maybe_session { struct building_block { building_block( const block_header_state& prev, block_timestamp_type when, + bool hotstuff_activated, uint16_t num_prev_blocks_to_confirm, const vector& new_protocol_feature_activations ) - :_pending_block_header_state( prev.next( when, num_prev_blocks_to_confirm ) ) + :_pending_block_header_state( prev.next( when, hotstuff_activated, num_prev_blocks_to_confirm ) ) ,_new_protocol_feature_activations( new_protocol_feature_activations ) ,_trx_mroot_or_receipt_digests( digests_t{} ) {} @@ -152,10 +153,11 @@ using block_stage_type = std::variant& new_protocol_feature_activations ) :_db_session( std::move(s) ) - ,_block_stage( building_block( prev, when, num_prev_blocks_to_confirm, new_protocol_feature_activations ) ) + ,_block_stage( building_block( prev, when, hotstuff_activated, num_prev_blocks_to_confirm, new_protocol_feature_activations ) ) {} maybe_session _db_session; @@ -235,6 +237,7 @@ struct controller_impl { std::optional pending; block_state_ptr head; fork_database fork_db; + std::atomic hs_irreversible_block_num{0}; resource_limits_manager resource_limits; subjective_billing subjective_bill; authorization_manager authorization; @@ -431,11 +434,13 @@ struct controller_impl { } const auto fork_head = fork_db_head(); + const uint32_t hs_lib = hs_irreversible_block_num; + const uint32_t new_lib = hs_lib > 0 ? hs_lib : fork_head->dpos_irreversible_blocknum; - if( fork_head->dpos_irreversible_blocknum <= lib_num ) + if( new_lib <= lib_num ) return; - auto branch = fork_db.fetch_branch( fork_head->id, fork_head->dpos_irreversible_blocknum ); + auto branch = fork_db.fetch_branch( fork_head->id, new_lib ); try { std::vector>> v; @@ -468,7 +473,7 @@ struct controller_impl { throw; } - //db.commit( fork_head->dpos_irreversible_blocknum ); // redundant + //db.commit( new_lib ); // redundant if( root_id != fork_db.root()->id ) { branch.emplace_back(fork_db.root()); @@ -1699,6 +1704,9 @@ struct controller_impl { { EOS_ASSERT( !pending, block_validate_exception, "pending block already exists" ); + // can change during start_block, so use same value throughout; although the transition from 0 to >0 cannot happen during start_block + uint32_t hs_lib = hs_irreversible_block_num.load(); + emit( self.block_start, head->block_num + 1 ); // at block level, no transaction specific logging is possible @@ -1716,9 +1724,9 @@ struct controller_impl { EOS_ASSERT( db.revision() == head->block_num, database_exception, "db revision is not on par with head block", ("db.revision()", db.revision())("controller_head_block", head->block_num)("fork_db_head_block", fork_db.head()->block_num) ); - pending.emplace( maybe_session(db), *head, when, confirm_block_count, new_protocol_feature_activations ); + pending.emplace( maybe_session(db), *head, when, hs_lib > 0, confirm_block_count, new_protocol_feature_activations ); } else { - pending.emplace( maybe_session(), *head, when, confirm_block_count, new_protocol_feature_activations ); + pending.emplace( maybe_session(), *head, when, hs_lib > 0, confirm_block_count, new_protocol_feature_activations ); } pending->_block_status = s; @@ -1800,22 +1808,23 @@ struct controller_impl { const auto& gpo = self.get_global_properties(); if( gpo.proposed_schedule_block_num && // if there is a proposed schedule that was proposed in a block ... - ( *gpo.proposed_schedule_block_num <= pbhs.dpos_irreversible_blocknum ) && // ... that has now become irreversible ... + ( hs_lib > 0 || *gpo.proposed_schedule_block_num <= pbhs.dpos_irreversible_blocknum ) && // ... that has now become irreversible or hotstuff activated... pbhs.prev_pending_schedule.schedule.producers.size() == 0 // ... and there was room for a new pending schedule prior to any possible promotion ) { - // Promote proposed schedule to pending schedule. + // Promote proposed schedule to pending schedule; happens in next block after hotstuff activated + EOS_ASSERT( gpo.proposed_schedule.version == pbhs.active_schedule_version + 1, + producer_schedule_exception, "wrong producer schedule version specified" ); + + std::get(pending->_block_stage)._new_pending_producer_schedule = producer_authority_schedule::from_shared(gpo.proposed_schedule); + if( !replaying ) { ilog( "promoting proposed schedule (set in block ${proposed_num}) to pending; current block: ${n} lib: ${lib} schedule: ${schedule} ", ("proposed_num", *gpo.proposed_schedule_block_num)("n", pbhs.block_num) - ("lib", pbhs.dpos_irreversible_blocknum) - ("schedule", producer_authority_schedule::from_shared(gpo.proposed_schedule) ) ); + ("lib", hs_lib > 0 ? hs_lib : pbhs.dpos_irreversible_blocknum) + ("schedule", std::get(pending->_block_stage)._new_pending_producer_schedule ) ); } - EOS_ASSERT( gpo.proposed_schedule.version == pbhs.active_schedule_version + 1, - producer_schedule_exception, "wrong producer schedule version specified" ); - - std::get(pending->_block_stage)._new_pending_producer_schedule = producer_authority_schedule::from_shared(gpo.proposed_schedule); db.modify( gpo, [&]( auto& gp ) { gp.proposed_schedule_block_num = std::optional(); gp.proposed_schedule.version=0; @@ -2212,6 +2221,7 @@ struct controller_impl { prev, b, protocol_features.get_protocol_feature_set(), + b->confirmed == hs_block_confirmed, // is hotstuff enabled for block [this]( block_timestamp_type timestamp, const flat_set& cur_features, const vector& new_features ) @@ -2318,6 +2328,7 @@ struct controller_impl { *head, b, protocol_features.get_protocol_feature_set(), + b->confirmed == hs_block_confirmed, // is hotstuff enabled for block [this]( block_timestamp_type timestamp, const flat_set& cur_features, const vector& new_features ) @@ -3168,6 +3179,11 @@ std::optional controller::pending_producer_block_id()const { return my->pending->_producer_block_id; } +void controller::set_hs_irreversible_block_num(uint32_t block_num) { + // needs to be set by qc_chain at startup and as irreversible changes + my->hs_irreversible_block_num = block_num; +} + uint32_t controller::last_irreversible_block_num() const { return my->fork_db.root()->block_num; } diff --git a/libraries/chain/fork_database.cpp b/libraries/chain/fork_database.cpp index 57bab920b2..52d6fcf397 100644 --- a/libraries/chain/fork_database.cpp +++ b/libraries/chain/fork_database.cpp @@ -42,6 +42,7 @@ namespace eosio { namespace chain { ordered_unique< tag, composite_key< block_state, global_fun, + // see first_preferred comment member, member, member @@ -57,6 +58,10 @@ namespace eosio { namespace chain { > fork_multi_index_type; bool first_preferred( const block_header_state& lhs, const block_header_state& rhs ) { + // dpos_irreversible_blocknum == std::numeric_limits::max() after hotstuff activation + // hotstuff block considered preferred over dpos + // hotstuff blocks compared by block_num as both lhs & rhs dpos_irreversible_blocknum is max uint32_t + // This can be simplified in a future release that assumes hotstuff already activated return std::tie( lhs.dpos_irreversible_blocknum, lhs.block_num ) > std::tie( rhs.dpos_irreversible_blocknum, rhs.block_num ); } diff --git a/libraries/chain/include/eosio/chain/block_header.hpp b/libraries/chain/include/eosio/chain/block_header.hpp index 8bb7976b76..b6dac4e2b1 100644 --- a/libraries/chain/include/eosio/chain/block_header.hpp +++ b/libraries/chain/include/eosio/chain/block_header.hpp @@ -25,6 +25,9 @@ namespace eosio { namespace chain { using block_header_extension = block_header_extension_types::block_header_extension_t; + // totem for block_header.confirmed that indicates hotstuff consensus is active + constexpr uint16_t hs_block_confirmed = std::numeric_limits::max(); + struct block_header { block_timestamp_type timestamp; @@ -38,6 +41,9 @@ namespace eosio { namespace chain { * No producer should sign a block with overlapping ranges or it is proof of byzantine * behavior. When producing a block a producer is always confirming at least the block he * is building off of. A producer cannot confirm "this" block, only prior blocks. + * + * After hotstuff activation a producer can no longer confirm blocks only propose them; + * confirmed will be std::numeric_limits::max() after hotstuff activation. */ uint16_t confirmed = 1; diff --git a/libraries/chain/include/eosio/chain/block_header_state.hpp b/libraries/chain/include/eosio/chain/block_header_state.hpp index 060c1ee943..d70390b90c 100644 --- a/libraries/chain/include/eosio/chain/block_header_state.hpp +++ b/libraries/chain/include/eosio/chain/block_header_state.hpp @@ -48,6 +48,10 @@ using signer_callback_type = std::function(const dig struct block_header_state; +// totem for dpos_irreversible_blocknum after hotstuff is activated +// This value implicitly means that fork_database will prefer hotstuff blocks over dpos blocks +constexpr uint32_t hs_dpos_irreversible_blocknum = std::numeric_limits::max(); + namespace detail { struct block_header_state_common { uint32_t block_num = 0; @@ -115,6 +119,33 @@ struct pending_block_header_state : public detail::block_header_state_common { /** * @struct block_header_state + * + * Algorithm for producer schedule change (pre-hostuff) + * privileged contract -> set_proposed_producers(producers) -> + * global_property_object.proposed_schedule_block_num = current_block_num + * global_property_object.proposed_schedule = producers + * + * start_block -> (global_property_object.proposed_schedule_block_num == dpos_lib) + * building_block._new_pending_producer_schedule = producers + * + * finalize_block -> + * block_header.extensions.wtmsig_block_signatures = producers + * block_header.new_producers = producers + * + * create_block_state -> + * block_state.schedule_lib_num = current_block_num + * block_state.pending_schedule.schedule = producers + * + * start_block -> + * block_state.prev_pending_schedule = pending_schedule (producers) + * if (pending_schedule.schedule_lib_num == dpos_lib) + * block_state.active_schedule = pending_schedule + * block_state.was_pending_promoted = true + * block_state.pending_schedule.clear() // doesn't get copied from previous + * else + * block_state.pending_schedule = prev_pending_schedule + * + * * @brief defines the minimum state necessary to validate transaction headers */ struct block_header_state : public detail::block_header_state_common { @@ -136,11 +167,12 @@ struct block_header_state : public detail::block_header_state_common { explicit block_header_state( legacy::snapshot_block_header_state_v2&& snapshot ); - pending_block_header_state next( block_timestamp_type when, uint16_t num_prev_blocks_to_confirm )const; + pending_block_header_state next( block_timestamp_type when, bool hotstuff_activated, uint16_t num_prev_blocks_to_confirm )const; block_header_state next( const signed_block_header& h, vector&& additional_signatures, const protocol_feature_set& pfs, + bool hotstuff_activated, const std::function&, const vector& )>& validator, diff --git a/libraries/chain/include/eosio/chain/block_state.hpp b/libraries/chain/include/eosio/chain/block_state.hpp index 4772094739..c8338a12fb 100644 --- a/libraries/chain/include/eosio/chain/block_state.hpp +++ b/libraries/chain/include/eosio/chain/block_state.hpp @@ -11,6 +11,7 @@ namespace eosio { namespace chain { block_state( const block_header_state& prev, signed_block_ptr b, const protocol_feature_set& pfs, + bool hotstuff_activated, const std::function&, const vector& )>& validator, diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index b86cba8a06..5db2b69495 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -239,6 +239,10 @@ namespace eosio { namespace chain { const producer_authority_schedule& pending_producers()const; std::optional proposed_producers()const; + // Called by qc_chain to indicate the current irreversible block num + // After hotstuff is activated, this should be called on startup by qc_chain + void set_hs_irreversible_block_num(uint32_t block_num); + uint32_t last_irreversible_block_num() const; block_id_type last_irreversible_block_id() const; time_point last_irreversible_block_time() const; diff --git a/libraries/hotstuff/chain_pacemaker.cpp b/libraries/hotstuff/chain_pacemaker.cpp index e034d9b1fc..32fbd63984 100644 --- a/libraries/hotstuff/chain_pacemaker.cpp +++ b/libraries/hotstuff/chain_pacemaker.cpp @@ -232,6 +232,10 @@ namespace eosio { namespace hotstuff { std::optional ext = blk->block->extract_header_extension(hs_finalizer_set_extension::extension_id()); if (ext) { std::scoped_lock g( _chain_state_mutex ); + if (_active_finalizer_set.generation == 0) { + // switching from dpos to hotstuff, all nodes will switch at same block height + _chain->set_hs_irreversible_block_num(blk->block_num); // can be any value <= dpos lib + } _active_finalizer_set = std::move(std::get(*ext)); } } diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index 757b930303..2af83154d0 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -765,18 +765,16 @@ class producer_plugin_impl : public std::enable_shared_from_thistimestamp < fc::minutes(5) || (blk_num % 1000 == 0)) { ilog("Received block ${id}... #${n} @ ${t} signed by ${p} " - "[trxs: ${count}, lib: ${lib}, confirmed: ${confs}, net: ${net}, cpu: ${cpu}, elapsed: ${elapsed}, time: ${time}, latency: " - "${latency} ms]", + "[trxs: ${count}, lib: ${lib}, net: ${net}, cpu: ${cpu}, elapsed: ${elapsed}, time: ${time}, latency: ${latency} ms]", ("p", block->producer)("id", id.str().substr(8, 16))("n", blk_num)("t", block->timestamp) ("count", block->transactions.size())("lib", chain.last_irreversible_block_num()) - ("confs", block->confirmed)("net", br.total_net_usage)("cpu", br.total_cpu_usage_us) + ("net", br.total_net_usage)("cpu", br.total_cpu_usage_us) ("elapsed", br.total_elapsed_time)("time", br.total_time)("latency", (now - block->timestamp).count() / 1000)); if (chain.get_read_mode() != db_read_mode::IRREVERSIBLE && hbs->id != id && hbs->block != nullptr) { // not applied to head ilog("Block not applied to head ${id}... #${n} @ ${t} signed by ${p} " - "[trxs: ${count}, dpos: ${dpos}, confirmed: ${confs}, net: ${net}, cpu: ${cpu}, elapsed: ${elapsed}, time: ${time}, " - "latency: ${latency} ms]", + "[trxs: ${count}, lib: ${lib}, net: ${net}, cpu: ${cpu}, elapsed: ${elapsed}, time: ${time}, latency: ${latency} ms]", ("p", hbs->block->producer)("id", hbs->id.str().substr(8, 16))("n", hbs->block_num)("t", hbs->block->timestamp) - ("count", hbs->block->transactions.size())("dpos", hbs->dpos_irreversible_blocknum)("confs", hbs->block->confirmed) + ("count", hbs->block->transactions.size())("lib", chain.last_irreversible_block_num()) ("net", br.total_net_usage)("cpu", br.total_cpu_usage_us)("elapsed", br.total_elapsed_time)("time", br.total_time) ("latency", (now - hbs->block->timestamp).count() / 1000)); } @@ -1911,11 +1909,11 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { try { uint16_t blocks_to_confirm = 0; - if (in_producing_mode()) { + if (in_producing_mode() && hbs->dpos_irreversible_blocknum != hs_dpos_irreversible_blocknum) { // determine how many blocks this producer can confirm // 1) if it is not a producer from this node, assume no confirmations (we will discard this block anyway) // 2) if it is a producer on this node that has never produced, the conservative approach is to assume no - // confirmations to make sure we don't double sign after a crash TODO: make these watermarks durable? + // confirmations to make sure we don't double sign after a crash // 3) if it is a producer on this node where this node knows the last block it produced, safely set it -UNLESS- // 4) the producer on this node's last watermark is higher (meaning on a different fork) if (current_watermark) { @@ -2857,6 +2855,8 @@ void producer_plugin_impl::switch_to_read_window() { _time_tracker.pause(); + fc_dlog(_log, "switch to read"); + // we are in write window, so no read-only trx threads are processing transactions. if (app().executor().read_only_queue().empty()) { // no read-only tasks to process. stay in write window start_write_window(); // restart write window timer for next round @@ -2881,6 +2881,7 @@ void producer_plugin_impl::switch_to_read_window() { _ro_exec_tasks_fut.emplace_back(post_async_task( _ro_thread_pool.get_executor(), [self = this, pending_block_num]() { return self->read_only_execution_task(pending_block_num); })); } + fc_dlog(_log, "read window"); auto expire_time = boost::posix_time::microseconds(_ro_read_window_time_us.count()); _ro_timer.expires_from_now(expire_time); From 718e17ddbbece5244630ea9b90a34a969c029bae Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 14 Sep 2023 13:13:30 -0500 Subject: [PATCH 2/5] GH-1523 GH-1631 Transition from dpos to hotstuff Add comment to schedule_lib_num. Do not want to rename as to not break clients that expect schedule_lib_num name. --- libraries/chain/include/eosio/chain/block_header_state.hpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/libraries/chain/include/eosio/chain/block_header_state.hpp b/libraries/chain/include/eosio/chain/block_header_state.hpp index d70390b90c..8fbdd6e57a 100644 --- a/libraries/chain/include/eosio/chain/block_header_state.hpp +++ b/libraries/chain/include/eosio/chain/block_header_state.hpp @@ -67,7 +67,10 @@ namespace detail { }; struct schedule_info { - uint32_t schedule_lib_num = 0; /// last irr block num + // schedule_lib_num is compared with dpos lib, but the value is actually current block at time of pending + // After hotstuff is activated, schedule_lib_num is compared to next().next() round for determination of + // changing from pending to active. + uint32_t schedule_lib_num = 0; /// block_num of pending digest_type schedule_hash; producer_authority_schedule schedule; }; @@ -133,7 +136,7 @@ struct pending_block_header_state : public detail::block_header_state_common { * block_header.new_producers = producers * * create_block_state -> - * block_state.schedule_lib_num = current_block_num + * block_state.schedule_lib_num = current_block_num (note this should be named schedule_block_num) * block_state.pending_schedule.schedule = producers * * start_block -> From 0b094688b0b4c6a2b2e15effc126a1a8b0aa6084 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 14 Sep 2023 13:15:36 -0500 Subject: [PATCH 3/5] Remove temp debug output --- plugins/producer_plugin/producer_plugin.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index 2af83154d0..dc48f03d47 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -2855,8 +2855,6 @@ void producer_plugin_impl::switch_to_read_window() { _time_tracker.pause(); - fc_dlog(_log, "switch to read"); - // we are in write window, so no read-only trx threads are processing transactions. if (app().executor().read_only_queue().empty()) { // no read-only tasks to process. stay in write window start_write_window(); // restart write window timer for next round @@ -2881,7 +2879,6 @@ void producer_plugin_impl::switch_to_read_window() { _ro_exec_tasks_fut.emplace_back(post_async_task( _ro_thread_pool.get_executor(), [self = this, pending_block_num]() { return self->read_only_execution_task(pending_block_num); })); } - fc_dlog(_log, "read window"); auto expire_time = boost::posix_time::microseconds(_ro_read_window_time_us.count()); _ro_timer.expires_from_now(expire_time); From 80de19e1df5ea14cb26eb532787b888749f639cb Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Tue, 7 Nov 2023 14:51:32 -0600 Subject: [PATCH 4/5] GH-1523 add additional comments --- libraries/chain/block_header_state.cpp | 13 ++++++++----- libraries/chain/controller.cpp | 1 + libraries/hotstuff/chain_pacemaker.cpp | 1 + plugins/producer_plugin/producer_plugin.cpp | 2 +- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/libraries/chain/block_header_state.cpp b/libraries/chain/block_header_state.cpp index 1d55254d6e..c1b8992776 100644 --- a/libraries/chain/block_header_state.cpp +++ b/libraries/chain/block_header_state.cpp @@ -17,9 +17,9 @@ namespace eosio { namespace chain { } uint32_t get_next_next_round_block_num( block_timestamp_type t, uint32_t block_num ) { - auto index = t.slot % config::producer_repetitions; - // remainder of current + next round - return block_num + config::producer_repetitions - index + config::producer_repetitions; + auto index = t.slot % config::producer_repetitions; // current index in current round + // (increment to the end of this round ) + next round + return block_num + (config::producer_repetitions - index) + config::producer_repetitions; } } @@ -43,6 +43,9 @@ namespace eosio { namespace chain { return blocknums[ index ]; } + // create pending_block_header_state from this for `when` + // If hotstuff_activated then use new consensus values and simpler active schedule update. + // If notstuff is not activated then use previous pre-hotstuff consensus logic. pending_block_header_state block_header_state::next( block_timestamp_type when, bool hotstuff_activated, uint16_t num_prev_blocks_to_confirm )const @@ -55,14 +58,14 @@ namespace eosio { namespace chain { (when = header.timestamp).slot++; } - auto proauth = get_scheduled_producer(when); - result.block_num = block_num + 1; result.previous = id; result.timestamp = when; result.active_schedule_version = active_schedule.version; result.prev_activated_protocol_features = activated_protocol_features; + auto proauth = get_scheduled_producer(when); + result.valid_block_signing_authority = proauth.authority; result.producer = proauth.producer_name; result.last_proposed_finalizer_set_generation = last_proposed_finalizer_set_generation; diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 4cde143371..f65ad54d60 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -3185,6 +3185,7 @@ std::optional controller::pending_producer_block_id()const { void controller::set_hs_irreversible_block_num(uint32_t block_num) { // needs to be set by qc_chain at startup and as irreversible changes + assert(block_num > 0); my->hs_irreversible_block_num = block_num; } diff --git a/libraries/hotstuff/chain_pacemaker.cpp b/libraries/hotstuff/chain_pacemaker.cpp index 43eeb49670..eac294893a 100644 --- a/libraries/hotstuff/chain_pacemaker.cpp +++ b/libraries/hotstuff/chain_pacemaker.cpp @@ -234,6 +234,7 @@ namespace eosio { namespace hotstuff { std::scoped_lock g( _chain_state_mutex ); if (_active_finalizer_set.generation == 0) { // switching from dpos to hotstuff, all nodes will switch at same block height + // block header extension is set in finalize_block to value set by host function set_finalizers _chain->set_hs_irreversible_block_num(blk->block_num); // can be any value <= dpos lib } _active_finalizer_set = std::move(std::get(*ext)); diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index 2c592610d5..4dd8ec631e 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -1862,7 +1862,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { try { uint16_t blocks_to_confirm = 0; - if (in_producing_mode() && hbs->dpos_irreversible_blocknum != hs_dpos_irreversible_blocknum) { + if (in_producing_mode() && hbs->dpos_irreversible_blocknum != hs_dpos_irreversible_blocknum) { // only if hotstuff not enabled // determine how many blocks this producer can confirm // 1) if it is not a producer from this node, assume no confirmations (we will discard this block anyway) // 2) if it is a producer on this node that has never produced, the conservative approach is to assume no From 7b22e57e623d07e15946ff7b36cd80e04f332c95 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 17 Nov 2023 09:51:10 -0600 Subject: [PATCH 5/5] GH-1523 add local variable hs_active to make code clearer --- libraries/chain/controller.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index f65ad54d60..1351bce3aa 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -1713,8 +1713,9 @@ struct controller_impl { { EOS_ASSERT( !pending, block_validate_exception, "pending block already exists" ); - // can change during start_block, so use same value throughout; although the transition from 0 to >0 cannot happen during start_block + // can change during start_block, so use same value throughout uint32_t hs_lib = hs_irreversible_block_num.load(); + const bool hs_active = hs_lib > 0; // the transition from 0 to >0 cannot happen during start_block emit( self.block_start, head->block_num + 1 ); @@ -1733,9 +1734,9 @@ struct controller_impl { EOS_ASSERT( db.revision() == head->block_num, database_exception, "db revision is not on par with head block", ("db.revision()", db.revision())("controller_head_block", head->block_num)("fork_db_head_block", fork_db.head()->block_num) ); - pending.emplace( maybe_session(db), *head, when, hs_lib > 0, confirm_block_count, new_protocol_feature_activations ); + pending.emplace( maybe_session(db), *head, when, hs_active, confirm_block_count, new_protocol_feature_activations ); } else { - pending.emplace( maybe_session(), *head, when, hs_lib > 0, confirm_block_count, new_protocol_feature_activations ); + pending.emplace( maybe_session(), *head, when, hs_active, confirm_block_count, new_protocol_feature_activations ); } pending->_block_status = s; @@ -1817,7 +1818,7 @@ struct controller_impl { const auto& gpo = self.get_global_properties(); if( gpo.proposed_schedule_block_num && // if there is a proposed schedule that was proposed in a block ... - ( hs_lib > 0 || *gpo.proposed_schedule_block_num <= pbhs.dpos_irreversible_blocknum ) && // ... that has now become irreversible or hotstuff activated... + ( hs_active || *gpo.proposed_schedule_block_num <= pbhs.dpos_irreversible_blocknum ) && // ... that has now become irreversible or hotstuff activated... pbhs.prev_pending_schedule.schedule.producers.size() == 0 // ... and there was room for a new pending schedule prior to any possible promotion ) { @@ -1830,7 +1831,7 @@ struct controller_impl { if( !replaying ) { ilog( "promoting proposed schedule (set in block ${proposed_num}) to pending; current block: ${n} lib: ${lib} schedule: ${schedule} ", ("proposed_num", *gpo.proposed_schedule_block_num)("n", pbhs.block_num) - ("lib", hs_lib > 0 ? hs_lib : pbhs.dpos_irreversible_blocknum) + ("lib", hs_active ? hs_lib : pbhs.dpos_irreversible_blocknum) ("schedule", std::get(pending->_block_stage)._new_pending_producer_schedule ) ); }