From 5f1e007d9ce77cb43ef757e8018b59623a7332d5 Mon Sep 17 00:00:00 2001 From: mtrela Date: Tue, 2 Jan 2018 16:18:35 +0100 Subject: [PATCH] Issue #1947 ( Follow/Tags Iterator Optimization ) --- libraries/plugins/follow/CMakeLists.txt | 1 + .../plugins/follow/follow_evaluators.cpp | 67 +++--- libraries/plugins/follow/follow_plugin.cpp | 54 ++--- libraries/plugins/follow/inc_performance.cpp | 216 ++++++++++++++++++ .../steem/plugins/follow/follow_objects.hpp | 57 +---- .../steem/plugins/follow/inc_performance.hpp | 133 +++++++++++ 6 files changed, 404 insertions(+), 124 deletions(-) create mode 100644 libraries/plugins/follow/inc_performance.cpp create mode 100644 libraries/plugins/follow/include/steem/plugins/follow/inc_performance.hpp diff --git a/libraries/plugins/follow/CMakeLists.txt b/libraries/plugins/follow/CMakeLists.txt index 020a5bf0e1..4a944385fa 100644 --- a/libraries/plugins/follow/CMakeLists.txt +++ b/libraries/plugins/follow/CMakeLists.txt @@ -4,6 +4,7 @@ add_library( follow_plugin follow_plugin.cpp follow_operations.cpp follow_evaluators.cpp + inc_performance.cpp ) target_link_libraries( follow_plugin chain_plugin ) diff --git a/libraries/plugins/follow/follow_evaluators.cpp b/libraries/plugins/follow/follow_evaluators.cpp index 6bbf0a94b3..e5d131bfd6 100644 --- a/libraries/plugins/follow/follow_evaluators.cpp +++ b/libraries/plugins/follow/follow_evaluators.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -121,6 +122,8 @@ void reblog_evaluator::do_apply( const reblog_operation& o ) { try { + performance perf( _db ); + const auto& c = _db.get_comment( o.author, o.permlink ); FC_ASSERT( c.parent_author.size() == 0, "Only top level posts can be reblogged" ); @@ -160,60 +163,52 @@ void reblog_evaluator::do_apply( const reblog_operation& o ) }); } - const auto& feed_idx = _db.get_index< feed_index >().indices().get< by_feed >(); const auto& comment_idx = _db.get_index< feed_index >().indices().get< by_comment >(); const auto& idx = _db.get_index< follow_index >().indices().get< by_following_follower >(); + const auto& old_feed_idx = _db.get_index< feed_index >().indices().get< by_feed >(); auto itr = idx.find( o.account ); + performance_data pd; + if( _db.head_block_time() >= _plugin->start_feeds ) { while( itr != idx.end() && itr->following == o.account ) { - if( itr->what & ( 1 << blog ) ) { - uint32_t next_id = 0; - auto last_feed = feed_idx.lower_bound( itr->follower ); - - if( last_feed != feed_idx.end() && last_feed->account == itr->follower ) - { - next_id = last_feed->account_feed_id + 1; - } - auto feed_itr = comment_idx.find( boost::make_tuple( c.id, itr->follower ) ); + bool is_empty = feed_itr == comment_idx.end(); - if( feed_itr == comment_idx.end() ) + pd.init( o.account, _db.head_block_time(), c.id, is_empty, is_empty ? 0 : feed_itr->account_feed_id ); + uint32_t next_id = perf.delete_old_objects< performance_data::t_creation_type::full_feed >( old_feed_idx, itr->follower, _plugin->max_feed_size, pd ); + + if( pd.s.creation ) { - _db.create< feed_object >( [&]( feed_object& f ) + if( is_empty ) { - f.account = itr->follower; - f.reblogged_by.push_back( o.account ); - f.first_reblogged_by = o.account; - f.first_reblogged_on = _db.head_block_time(); - f.comment = c.id; - f.reblogs = 1; - f.account_feed_id = next_id; - }); - } - else - { - _db.modify( *feed_itr, [&]( feed_object& f ) + _db.create< feed_object >( [&]( feed_object& f ) + { + f.account = itr->follower; + f.reblogged_by.push_back( o.account ); + f.first_reblogged_by = o.account; + f.first_reblogged_on = _db.head_block_time(); + f.comment = c.id; + f.account_feed_id = next_id; + }); + } + else { - f.reblogged_by.push_back( o.account ); - f.reblogs++; - }); + if( pd.s.allow_modify ) + { + _db.modify( *feed_itr, [&]( feed_object& f ) + { + f.reblogged_by.push_back( o.account ); + }); + } + } } - const auto& old_feed_idx = _db.get_index< feed_index >().indices().get< by_old_feed >(); - auto old_feed = old_feed_idx.lower_bound( itr->follower ); - - while( old_feed->account == itr->follower && next_id - old_feed->account_feed_id > _plugin->max_feed_size ) - { - _db.remove( *old_feed ); - old_feed = old_feed_idx.lower_bound( itr->follower ); - }; } - ++itr; } } diff --git a/libraries/plugins/follow/follow_plugin.cpp b/libraries/plugins/follow/follow_plugin.cpp index 04ff94cf09..ff92b78849 100644 --- a/libraries/plugins/follow/follow_plugin.cpp +++ b/libraries/plugins/follow/follow_plugin.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include @@ -135,8 +136,10 @@ struct post_operation_visitor { follow_plugin_impl& _plugin; + performance perf; + post_operation_visitor( follow_plugin_impl& plugin ) - : _plugin( plugin ) {} + : _plugin( plugin ), perf( plugin._db ) {} typedef void result_type; @@ -186,9 +189,10 @@ struct post_operation_visitor const auto& idx = db.get_index< follow_index >().indices().get< by_following_follower >(); const auto& comment_idx = db.get_index< feed_index >().indices().get< by_comment >(); + const auto& old_feed_idx = db.get_index< feed_index >().indices().get< by_feed >(); auto itr = idx.find( op.author ); - const auto& feed_idx = db.get_index< feed_index >().indices().get< by_feed >(); + performance_data pd; if( db.head_block_time() >= _plugin._self.start_feeds ) { @@ -196,15 +200,13 @@ struct post_operation_visitor { if( itr->what & ( 1 << blog ) ) { - uint32_t next_id = 0; - auto last_feed = feed_idx.lower_bound( itr->follower ); + auto feed_itr = comment_idx.find( boost::make_tuple( c.id, itr->follower ) ); + bool is_empty = feed_itr == comment_idx.end(); - if( last_feed != feed_idx.end() && last_feed->account == itr->follower ) - { - next_id = last_feed->account_feed_id + 1; - } + pd.init( c.id, is_empty ); + uint32_t next_id = perf.delete_old_objects< performance_data::t_creation_type::part_feed >( old_feed_idx, itr->follower, _plugin._self.max_feed_size, pd ); - if( comment_idx.find( boost::make_tuple( c.id, itr->follower ) ) == comment_idx.end() ) + if( pd.s.creation && is_empty ) { db.create< feed_object >( [&]( feed_object& f ) { @@ -212,33 +214,22 @@ struct post_operation_visitor f.comment = c.id; f.account_feed_id = next_id; }); - - const auto& old_feed_idx = db.get_index< feed_index >().indices().get< by_old_feed >(); - auto old_feed = old_feed_idx.lower_bound( itr->follower ); - - while( old_feed->account == itr->follower && next_id - old_feed->account_feed_id > _plugin._self.max_feed_size ) - { - db.remove( *old_feed ); - old_feed = old_feed_idx.lower_bound( itr->follower ); - } } - } + } ++itr; } } - const auto& blog_idx = db.get_index< blog_index >().indices().get< by_blog >(); const auto& comment_blog_idx = db.get_index< blog_index >().indices().get< by_comment >(); - auto last_blog = blog_idx.lower_bound( op.author ); - uint32_t next_id = 0; + auto blog_itr = comment_blog_idx.find( boost::make_tuple( c.id, op.author ) ); + const auto& old_blog_idx = db.get_index< blog_index >().indices().get< by_blog >(); + bool is_empty = blog_itr == comment_blog_idx.end(); - if( last_blog != blog_idx.end() && last_blog->account == op.author ) - { - next_id = last_blog->blog_feed_id + 1; - } + pd.init( c.id, is_empty ); + uint32_t next_id = perf.delete_old_objects< performance_data::t_creation_type::full_blog >( old_blog_idx, op.author, _plugin._self.max_feed_size, pd ); - if( comment_blog_idx.find( boost::make_tuple( c.id, op.author ) ) == comment_blog_idx.end() ) + if( pd.s.creation && is_empty ) { db.create< blog_object >( [&]( blog_object& b) { @@ -246,15 +237,6 @@ struct post_operation_visitor b.comment = c.id; b.blog_feed_id = next_id; }); - - const auto& old_blog_idx = db.get_index< blog_index >().indices().get< by_old_blog >(); - auto old_blog = old_blog_idx.lower_bound( op.author ); - - while( old_blog->account == op.author && next_id - old_blog->blog_feed_id > _plugin._self.max_feed_size ) - { - db.remove( *old_blog ); - old_blog = old_blog_idx.lower_bound( op.author ); - } } } FC_LOG_AND_RETHROW() diff --git a/libraries/plugins/follow/inc_performance.cpp b/libraries/plugins/follow/inc_performance.cpp new file mode 100644 index 0000000000..fa0b101b50 --- /dev/null +++ b/libraries/plugins/follow/inc_performance.cpp @@ -0,0 +1,216 @@ +#include + +#include +#include + +namespace steem { namespace plugins{ namespace follow { + +std::unique_ptr< dumper > dumper::self; + +class performance_impl +{ + database& db; + + template< typename Object > + uint32_t get_actual_id( const Object& it ) const; + + template< typename Object > + const char* get_actual_name( const Object& it ) const; + + template< performance_data::t_creation_type CreationType, typename Object > + void modify( const Object& obj, const account_name_type& start_account, uint32_t next_id, performance_data& pd ) const; + + template< typename Iterator > + void skip_modify( const Iterator& actual, performance_data& pd ) const; + + template< performance_data::t_creation_type CreationType, typename Iterator > + void remember_last( bool is_delayed, bool& init, Iterator& actual, performance_data& pd ) const; + + public: + + performance_impl( database& _db ); + ~performance_impl(); + + template< performance_data::t_creation_type CreationType, typename Index > + uint32_t delete_old_objects( const Index& old_idx, const account_name_type& start_account, uint32_t max_size, performance_data& pd ) const; +}; + + +performance_impl::performance_impl( database& _db ) + : db( _db ) +{ +} + +performance_impl::~performance_impl() +{ + +} + +template<> +uint32_t performance_impl::get_actual_id( const feed_object& obj ) const +{ + return obj.account_feed_id; +} + +template<> +uint32_t performance_impl::get_actual_id( const blog_object& obj ) const +{ + return obj.blog_feed_id; +} + +template<> +const char* performance_impl::get_actual_name( const feed_object& obj ) const +{ + return "feed"; +} + +template<> +const char* performance_impl::get_actual_name( const blog_object& obj ) const +{ + return "blog"; +} + +template<> +void performance_impl::modify< performance_data::t_creation_type::full_feed >( const feed_object& obj, const account_name_type& start_account, uint32_t next_id, performance_data& pd ) const +{ + pd.s.creation = false; + + db.modify( obj, [&]( feed_object& f ) + { + f.account = start_account; + + if( f.reblogged_by.size() == 1 ) + f.reblogged_by[0] = *pd.account; + else + { + f.reblogged_by.clear(); + f.reblogged_by.push_back( *pd.account ); + } + + f.first_reblogged_by = *pd.account; + f.first_reblogged_on = *pd.time; + f.comment = *pd.comment; + f.account_feed_id = next_id; + }); +} + +template<> +void performance_impl::modify< performance_data::t_creation_type::part_feed >( const feed_object& obj, const account_name_type& start_account, uint32_t next_id, performance_data& pd ) const +{ + pd.s.creation = false; + + db.modify( obj, [&]( feed_object& f ) + { + f.account = start_account; + f.reblogged_by.clear(); + f.comment = *pd.comment; + f.account_feed_id = next_id; + f.first_reblogged_by = account_name_type(); + f.first_reblogged_on = time_point_sec(); + }); + +} + +template<> +void performance_impl::modify< performance_data::t_creation_type::full_blog >( const blog_object& obj, const account_name_type& start_account, uint32_t next_id, performance_data& pd ) const +{ + pd.s.creation = false; + + db.modify( obj, [&]( blog_object& b ) + { + b.account = start_account; + b.comment = *pd.comment; + b.blog_feed_id = next_id; + b.reblogged_on = time_point_sec(); + }); +} + +template< typename Iterator > +void performance_impl::skip_modify( const Iterator& actual, performance_data& pd ) const +{ + uint32_t _id = get_actual_id( *actual ); + if( _id == pd.old_id ) + { + pd.s.allow_modify = false; + } +} + +template< performance_data::t_creation_type CreationType, typename Iterator > +void performance_impl::remember_last( bool is_delayed, bool& init, Iterator& actual, performance_data& pd ) const +{ + if( is_delayed ) + { + if( init ) + init = false; + else + { + auto removed = std::prev( actual ); + if( CreationType == performance_data::t_creation_type::full_feed ) + skip_modify( removed, pd ); + db.remove( *removed ); + } + } + else + { + if( CreationType == performance_data::t_creation_type::full_feed ) + skip_modify( actual, pd ); + db.remove( *actual ); + } +} + +template< performance_data::t_creation_type CreationType, typename Index > +uint32_t performance_impl::delete_old_objects( const Index& old_idx, const account_name_type& start_account, uint32_t max_size, performance_data& pd ) const +{ + auto it_l = old_idx.lower_bound( start_account ); + auto it_u = old_idx.upper_bound( start_account ); + + if( it_l == it_u ) + return 0; + + uint32_t next_id = get_actual_id( *it_l ) + 1; + + auto r_end = old_idx.rend(); + decltype( r_end ) it( it_u ); + + bool is_init = true; + + while( it != r_end && it->account == start_account && next_id - get_actual_id( *it ) > max_size ) + { + auto old_itr = it; + ++it; + + remember_last< CreationType >( pd.s.is_empty, is_init, old_itr, pd ); + } + + if( !is_init ) + modify< CreationType >( *std::prev( it ), start_account, next_id, pd ); + + return next_id; +} + +performance::performance( database& _db ) + : my( new performance_impl( _db ) ) +{ + +} + +performance::~performance() +{ + +} + +template< performance_data::t_creation_type CreationType, typename Index > +uint32_t performance::delete_old_objects( const Index& old_idx, const account_name_type& start_account, uint32_t max_size, performance_data& pd ) const +{ + FC_ASSERT( my ); + return my->delete_old_objects< CreationType >( old_idx, start_account, max_size, pd ); +} + +using t_feed = decltype( ((database*)nullptr)->get_index< feed_index >().indices().get< by_feed >() ); +using t_blog = decltype( ((database*)nullptr)->get_index< blog_index >().indices().get< by_blog >() ); + +template uint32_t performance::delete_old_objects< performance_data::t_creation_type::full_feed >( t_feed& old_idx, const account_name_type& start_account, uint32_t max_size, performance_data& pd ) const; +template uint32_t performance::delete_old_objects< performance_data::t_creation_type::part_feed >( t_feed& old_idx, const account_name_type& start_account, uint32_t max_size, performance_data& pd ) const; +template uint32_t performance::delete_old_objects< performance_data::t_creation_type::full_blog >( t_blog& old_idx, const account_name_type& start_account, uint32_t max_size, performance_data& pd ) const; + +} } } //steem::follow diff --git a/libraries/plugins/follow/include/steem/plugins/follow/follow_objects.hpp b/libraries/plugins/follow/include/steem/plugins/follow/follow_objects.hpp index 487f683f7b..8a38e8385c 100644 --- a/libraries/plugins/follow/include/steem/plugins/follow/follow_objects.hpp +++ b/libraries/plugins/follow/include/steem/plugins/follow/follow_objects.hpp @@ -55,6 +55,8 @@ typedef oid< follow_object > follow_id_type; class feed_object : public object< feed_object_type, feed_object > { public: + typedef t_vector t_reblogged_by_container; + feed_object() = delete; template< typename Constructor, typename Allocator > @@ -64,16 +66,13 @@ class feed_object : public object< feed_object_type, feed_object > c( *this ); } - typedef t_vector t_reblogged_by_container; - id_type id; account_name_type account; - t_vector reblogged_by; + t_reblogged_by_container reblogged_by; account_name_type first_reblogged_by; time_point_sec first_reblogged_on; comment_id_type comment; - uint32_t reblogs; uint32_t account_feed_id = 0; }; @@ -212,7 +211,6 @@ typedef chainbase::shared_multi_index_container< > blog_author_stats_index; struct by_feed; -struct by_old_feed; struct by_account; struct by_comment; @@ -227,20 +225,6 @@ typedef multi_index_container< >, composite_key_compare< std::less< account_name_type >, std::greater< uint32_t > > >, - ordered_unique< tag< by_old_feed >, - composite_key< feed_object, - member< feed_object, account_name_type, &feed_object::account >, - member< feed_object, uint32_t, &feed_object::account_feed_id > - >, - composite_key_compare< std::less< account_name_type >, std::less< uint32_t > > - >, - ordered_unique< tag< by_account >, - composite_key< feed_object, - member< feed_object, account_name_type, &feed_object::account >, - member< feed_object, feed_id_type, &feed_object::id > - >, - composite_key_compare< std::less< account_name_type >, std::less< feed_id_type > > - >, ordered_unique< tag< by_comment >, composite_key< feed_object, member< feed_object, comment_id_type, &feed_object::comment >, @@ -253,7 +237,6 @@ typedef multi_index_container< > feed_index; struct by_blog; -struct by_old_blog; typedef multi_index_container< blog_object, @@ -266,13 +249,6 @@ typedef multi_index_container< >, composite_key_compare< std::less< account_name_type >, std::greater< uint32_t > > >, - ordered_unique< tag< by_old_blog >, - composite_key< blog_object, - member< blog_object, account_name_type, &blog_object::account >, - member< blog_object, uint32_t, &blog_object::blog_feed_id > - >, - composite_key_compare< std::less< account_name_type >, std::less< uint32_t > > - >, ordered_unique< tag< by_comment >, composite_key< blog_object, member< blog_object, comment_id_type, &blog_object::comment >, @@ -284,19 +260,10 @@ typedef multi_index_container< allocator< blog_object > > blog_index; -struct by_reputation; - typedef multi_index_container< reputation_object, indexed_by< ordered_unique< tag< by_id >, member< reputation_object, reputation_id_type, &reputation_object::id > >, - ordered_unique< tag< by_reputation >, - composite_key< reputation_object, - member< reputation_object, share_type, &reputation_object::reputation >, - member< reputation_object, account_name_type, &reputation_object::account > - >, - composite_key_compare< std::greater< share_type >, std::less< account_name_type > > - >, ordered_unique< tag< by_account >, member< reputation_object, account_name_type, &reputation_object::account > > >, allocator< reputation_object > @@ -310,21 +277,7 @@ typedef multi_index_container< follow_count_object, indexed_by< ordered_unique< tag< by_id >, member< follow_count_object, follow_count_id_type, &follow_count_object::id > >, - ordered_unique< tag< by_account >, member< follow_count_object, account_name_type, &follow_count_object::account > >, - ordered_unique< tag< by_followers >, - composite_key< follow_count_object, - member< follow_count_object, uint32_t, &follow_count_object::follower_count >, - member< follow_count_object, follow_count_id_type, &follow_count_object::id > - >, - composite_key_compare< std::greater< uint32_t >, std::less< follow_count_id_type > > - >, - ordered_unique< tag< by_following >, - composite_key< follow_count_object, - member< follow_count_object, uint32_t, &follow_count_object::following_count >, - member< follow_count_object, follow_count_id_type, &follow_count_object::id > - >, - composite_key_compare< std::greater< uint32_t >, std::less< follow_count_id_type > > - > + ordered_unique< tag< by_account >, member< follow_count_object, account_name_type, &follow_count_object::account > > >, allocator< follow_count_object > > follow_count_index; @@ -336,7 +289,7 @@ FC_REFLECT_ENUM( steem::plugins::follow::follow_type, (undefined)(blog)(ignore) FC_REFLECT( steem::plugins::follow::follow_object, (id)(follower)(following)(what) ) CHAINBASE_SET_INDEX_TYPE( steem::plugins::follow::follow_object, steem::plugins::follow::follow_index ) -FC_REFLECT( steem::plugins::follow::feed_object, (id)(account)(first_reblogged_by)(first_reblogged_on)(reblogged_by)(comment)(reblogs)(account_feed_id) ) +FC_REFLECT( steem::plugins::follow::feed_object, (id)(account)(first_reblogged_by)(first_reblogged_on)(reblogged_by)(comment)(account_feed_id) ) CHAINBASE_SET_INDEX_TYPE( steem::plugins::follow::feed_object, steem::plugins::follow::feed_index ) FC_REFLECT( steem::plugins::follow::blog_object, (id)(account)(comment)(reblogged_on)(blog_feed_id) ) diff --git a/libraries/plugins/follow/include/steem/plugins/follow/inc_performance.hpp b/libraries/plugins/follow/include/steem/plugins/follow/inc_performance.hpp new file mode 100644 index 0000000000..f463410361 --- /dev/null +++ b/libraries/plugins/follow/include/steem/plugins/follow/inc_performance.hpp @@ -0,0 +1,133 @@ +#pragma once + +#include +#include +#include + +namespace steem { namespace plugins{ namespace follow { + +using namespace steem::chain; +using steem::chain::database; + +using steem::protocol::account_name_type; + +class performance_impl; + +class dumper +{ + private: + + std::ofstream f; + + static std::unique_ptr< dumper > self; + + dumper() : +#if ENABLE_STD_ALLOCATOR == 1 + f( "std_dumped_objects.txt" ) +#else + f( "bip_dumped_objects.txt" ) +#endif + { + } + + public: + + ~dumper() + { + f.flush(); + f.close(); + } + + static std::unique_ptr< dumper >& instance() + { + if( !self ) + self = std::unique_ptr< dumper >( new dumper() ); + + return self; + } + + template< typename T, typename T2 > + void dump( const char* message, const T& data, const T2& data2 ) + { + static uint64_t counter = 0; + f< my; + + public: + + performance( database& _db ); + ~performance(); + + template< performance_data::t_creation_type CreationType, typename Index > + uint32_t delete_old_objects( const Index& old_idx, const account_name_type& start_account, uint32_t max_size, performance_data& pd ) const; + + template< typename T, typename T2 > + static void dump( const char* message, const T& data, const T2& data2 ) + { + dumper::instance()->dump( message, data, data2 ); + } +}; + +} } } //steem::follow