From 5d04907a1fd30c91b110f663bf1164c7bb323d54 Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Tue, 16 Jan 2018 09:19:30 -0800 Subject: [PATCH] Revert "Issue #1947 ( Follow/Tags Iterator Optimization )" --- libraries/plugins/follow/CMakeLists.txt | 1 - .../plugins/follow/follow_evaluators.cpp | 67 +++--- libraries/plugins/follow/follow_plugin.cpp | 54 +++-- libraries/plugins/follow/inc_performance.cpp | 216 ------------------ .../steem/plugins/follow/follow_objects.hpp | 62 ++++- .../steem/plugins/follow/inc_performance.hpp | 133 ----------- 6 files changed, 124 insertions(+), 409 deletions(-) delete mode 100644 libraries/plugins/follow/inc_performance.cpp delete mode 100644 libraries/plugins/follow/include/steem/plugins/follow/inc_performance.hpp diff --git a/libraries/plugins/follow/CMakeLists.txt b/libraries/plugins/follow/CMakeLists.txt index 4a944385fa..020a5bf0e1 100644 --- a/libraries/plugins/follow/CMakeLists.txt +++ b/libraries/plugins/follow/CMakeLists.txt @@ -4,7 +4,6 @@ add_library( follow_plugin follow_plugin.cpp follow_operations.cpp follow_evaluators.cpp - inc_performance.cpp ) target_link_libraries( follow_plugin chain_plugin ) diff --git a/libraries/plugins/follow/follow_evaluators.cpp b/libraries/plugins/follow/follow_evaluators.cpp index e5d131bfd6..6bbf0a94b3 100644 --- a/libraries/plugins/follow/follow_evaluators.cpp +++ b/libraries/plugins/follow/follow_evaluators.cpp @@ -1,7 +1,6 @@ #include #include #include -#include #include #include @@ -122,8 +121,6 @@ void reblog_evaluator::do_apply( const reblog_operation& o ) { try { - performance perf( _db ); - const auto& c = _db.get_comment( o.author, o.permlink ); FC_ASSERT( c.parent_author.size() == 0, "Only top level posts can be reblogged" ); @@ -163,52 +160,60 @@ void reblog_evaluator::do_apply( const reblog_operation& o ) }); } + const auto& feed_idx = _db.get_index< feed_index >().indices().get< by_feed >(); const auto& comment_idx = _db.get_index< feed_index >().indices().get< by_comment >(); const auto& idx = _db.get_index< follow_index >().indices().get< by_following_follower >(); - const auto& old_feed_idx = _db.get_index< feed_index >().indices().get< by_feed >(); auto itr = idx.find( o.account ); - performance_data pd; - if( _db.head_block_time() >= _plugin->start_feeds ) { while( itr != idx.end() && itr->following == o.account ) { + if( itr->what & ( 1 << blog ) ) { - auto feed_itr = comment_idx.find( boost::make_tuple( c.id, itr->follower ) ); - bool is_empty = feed_itr == comment_idx.end(); + uint32_t next_id = 0; + auto last_feed = feed_idx.lower_bound( itr->follower ); - pd.init( o.account, _db.head_block_time(), c.id, is_empty, is_empty ? 0 : feed_itr->account_feed_id ); - uint32_t next_id = perf.delete_old_objects< performance_data::t_creation_type::full_feed >( old_feed_idx, itr->follower, _plugin->max_feed_size, pd ); + if( last_feed != feed_idx.end() && last_feed->account == itr->follower ) + { + next_id = last_feed->account_feed_id + 1; + } - if( pd.s.creation ) + auto feed_itr = comment_idx.find( boost::make_tuple( c.id, itr->follower ) ); + + if( feed_itr == comment_idx.end() ) { - if( is_empty ) + _db.create< feed_object >( [&]( feed_object& f ) { - _db.create< feed_object >( [&]( feed_object& f ) - { - f.account = itr->follower; - f.reblogged_by.push_back( o.account ); - f.first_reblogged_by = o.account; - f.first_reblogged_on = _db.head_block_time(); - f.comment = c.id; - f.account_feed_id = next_id; - }); - } - else + f.account = itr->follower; + f.reblogged_by.push_back( o.account ); + f.first_reblogged_by = o.account; + f.first_reblogged_on = _db.head_block_time(); + f.comment = c.id; + f.reblogs = 1; + f.account_feed_id = next_id; + }); + } + else + { + _db.modify( *feed_itr, [&]( feed_object& f ) { - if( pd.s.allow_modify ) - { - _db.modify( *feed_itr, [&]( feed_object& f ) - { - f.reblogged_by.push_back( o.account ); - }); - } - } + f.reblogged_by.push_back( o.account ); + f.reblogs++; + }); } + const auto& old_feed_idx = _db.get_index< feed_index >().indices().get< by_old_feed >(); + auto old_feed = old_feed_idx.lower_bound( itr->follower ); + + while( old_feed->account == itr->follower && next_id - old_feed->account_feed_id > _plugin->max_feed_size ) + { + _db.remove( *old_feed ); + old_feed = old_feed_idx.lower_bound( itr->follower ); + }; } + ++itr; } } diff --git a/libraries/plugins/follow/follow_plugin.cpp b/libraries/plugins/follow/follow_plugin.cpp index ff92b78849..04ff94cf09 100644 --- a/libraries/plugins/follow/follow_plugin.cpp +++ b/libraries/plugins/follow/follow_plugin.cpp @@ -1,7 +1,6 @@ #include #include #include -#include #include @@ -136,10 +135,8 @@ struct post_operation_visitor { follow_plugin_impl& _plugin; - performance perf; - post_operation_visitor( follow_plugin_impl& plugin ) - : _plugin( plugin ), perf( plugin._db ) {} + : _plugin( plugin ) {} typedef void result_type; @@ -189,10 +186,9 @@ struct post_operation_visitor const auto& idx = db.get_index< follow_index >().indices().get< by_following_follower >(); const auto& comment_idx = db.get_index< feed_index >().indices().get< by_comment >(); - const auto& old_feed_idx = db.get_index< feed_index >().indices().get< by_feed >(); auto itr = idx.find( op.author ); - performance_data pd; + const auto& feed_idx = db.get_index< feed_index >().indices().get< by_feed >(); if( db.head_block_time() >= _plugin._self.start_feeds ) { @@ -200,13 +196,15 @@ struct post_operation_visitor { if( itr->what & ( 1 << blog ) ) { - auto feed_itr = comment_idx.find( boost::make_tuple( c.id, itr->follower ) ); - bool is_empty = feed_itr == comment_idx.end(); + uint32_t next_id = 0; + auto last_feed = feed_idx.lower_bound( itr->follower ); - pd.init( c.id, is_empty ); - uint32_t next_id = perf.delete_old_objects< performance_data::t_creation_type::part_feed >( old_feed_idx, itr->follower, _plugin._self.max_feed_size, pd ); + if( last_feed != feed_idx.end() && last_feed->account == itr->follower ) + { + next_id = last_feed->account_feed_id + 1; + } - if( pd.s.creation && is_empty ) + if( comment_idx.find( boost::make_tuple( c.id, itr->follower ) ) == comment_idx.end() ) { db.create< feed_object >( [&]( feed_object& f ) { @@ -214,22 +212,33 @@ struct post_operation_visitor f.comment = c.id; f.account_feed_id = next_id; }); - } + const auto& old_feed_idx = db.get_index< feed_index >().indices().get< by_old_feed >(); + auto old_feed = old_feed_idx.lower_bound( itr->follower ); + + while( old_feed->account == itr->follower && next_id - old_feed->account_feed_id > _plugin._self.max_feed_size ) + { + db.remove( *old_feed ); + old_feed = old_feed_idx.lower_bound( itr->follower ); + } + } } + ++itr; } } + const auto& blog_idx = db.get_index< blog_index >().indices().get< by_blog >(); const auto& comment_blog_idx = db.get_index< blog_index >().indices().get< by_comment >(); - auto blog_itr = comment_blog_idx.find( boost::make_tuple( c.id, op.author ) ); - const auto& old_blog_idx = db.get_index< blog_index >().indices().get< by_blog >(); - bool is_empty = blog_itr == comment_blog_idx.end(); + auto last_blog = blog_idx.lower_bound( op.author ); + uint32_t next_id = 0; - pd.init( c.id, is_empty ); - uint32_t next_id = perf.delete_old_objects< performance_data::t_creation_type::full_blog >( old_blog_idx, op.author, _plugin._self.max_feed_size, pd ); + if( last_blog != blog_idx.end() && last_blog->account == op.author ) + { + next_id = last_blog->blog_feed_id + 1; + } - if( pd.s.creation && is_empty ) + if( comment_blog_idx.find( boost::make_tuple( c.id, op.author ) ) == comment_blog_idx.end() ) { db.create< blog_object >( [&]( blog_object& b) { @@ -237,6 +246,15 @@ struct post_operation_visitor b.comment = c.id; b.blog_feed_id = next_id; }); + + const auto& old_blog_idx = db.get_index< blog_index >().indices().get< by_old_blog >(); + auto old_blog = old_blog_idx.lower_bound( op.author ); + + while( old_blog->account == op.author && next_id - old_blog->blog_feed_id > _plugin._self.max_feed_size ) + { + db.remove( *old_blog ); + old_blog = old_blog_idx.lower_bound( op.author ); + } } } FC_LOG_AND_RETHROW() diff --git a/libraries/plugins/follow/inc_performance.cpp b/libraries/plugins/follow/inc_performance.cpp deleted file mode 100644 index 7a656be3ec..0000000000 --- a/libraries/plugins/follow/inc_performance.cpp +++ /dev/null @@ -1,216 +0,0 @@ -#include - -#include -#include - -namespace steem { namespace plugins{ namespace follow { - -std::unique_ptr< dumper > dumper::self; - -class performance_impl -{ - database& db; - - template< typename Object > - uint32_t get_actual_id( const Object& it ) const; - - template< typename Object > - const char* get_actual_name( const Object& it ) const; - - template< performance_data::t_creation_type CreationType, typename Object > - void modify( const Object& obj, const account_name_type& start_account, uint32_t next_id, performance_data& pd ) const; - - template< typename Iterator > - void skip_modify( const Iterator& actual, performance_data& pd ) const; - - template< performance_data::t_creation_type CreationType, typename Iterator > - void remember_last( bool is_delayed, bool& init, Iterator& actual, performance_data& pd ) const; - - public: - - performance_impl( database& _db ); - ~performance_impl(); - - template< performance_data::t_creation_type CreationType, typename Index > - uint32_t delete_old_objects( const Index& old_idx, const account_name_type& start_account, uint32_t max_size, performance_data& pd ) const; -}; - - -performance_impl::performance_impl( database& _db ) - : db( _db ) -{ -} - -performance_impl::~performance_impl() -{ - -} - -template<> -uint32_t performance_impl::get_actual_id( const feed_object& obj ) const -{ - return obj.account_feed_id; -} - -template<> -uint32_t performance_impl::get_actual_id( const blog_object& obj ) const -{ - return obj.blog_feed_id; -} - -template<> -const char* performance_impl::get_actual_name( const feed_object& obj ) const -{ - return "feed"; -} - -template<> -const char* performance_impl::get_actual_name( const blog_object& obj ) const -{ - return "blog"; -} - -template<> -void performance_impl::modify< performance_data::t_creation_type::full_feed >( const feed_object& obj, const account_name_type& start_account, uint32_t next_id, performance_data& pd ) const -{ - pd.s.creation = false; - - db.modify( obj, [&]( feed_object& f ) - { - f.account = start_account; - - if( f.reblogged_by.size() == 1 ) - f.reblogged_by[0] = *pd.account; - else - { - f.reblogged_by.clear(); - f.reblogged_by.push_back( *pd.account ); - } - - f.first_reblogged_by = *pd.account; - f.first_reblogged_on = *pd.time; - f.comment = *pd.comment; - f.account_feed_id = next_id; - }); -} - -template<> -void performance_impl::modify< performance_data::t_creation_type::part_feed >( const feed_object& obj, const account_name_type& start_account, uint32_t next_id, performance_data& pd ) const -{ - pd.s.creation = false; - - db.modify( obj, [&]( feed_object& f ) - { - f.account = start_account; - f.reblogged_by.clear(); - f.comment = *pd.comment; - f.account_feed_id = next_id; - f.first_reblogged_by = account_name_type(); - f.first_reblogged_on = time_point_sec(); - }); - -} - -template<> -void performance_impl::modify< performance_data::t_creation_type::full_blog >( const blog_object& obj, const account_name_type& start_account, uint32_t next_id, performance_data& pd ) const -{ - pd.s.creation = false; - - db.modify( obj, [&]( blog_object& b ) - { - b.account = start_account; - b.comment = *pd.comment; - b.blog_feed_id = next_id; - b.reblogged_on = time_point_sec(); - }); -} - -template< typename Iterator > -void performance_impl::skip_modify( const Iterator& actual, performance_data& pd ) const -{ - uint32_t _id = get_actual_id( *actual ); - if( _id == pd.old_id ) - { - pd.s.allow_modify = false; - } -} - -template< performance_data::t_creation_type CreationType, typename Iterator > -void performance_impl::remember_last( bool is_delayed, bool& init, Iterator& actual, performance_data& pd ) const -{ - if( is_delayed ) - { - if( init ) - init = false; - else - { - auto removed = std::prev( actual ); - if( CreationType == performance_data::t_creation_type::full_feed ) - skip_modify( removed, pd ); - db.remove( *removed ); - } - } - else - { - if( CreationType == performance_data::t_creation_type::full_feed ) - skip_modify( actual, pd ); - db.remove( *actual ); - } -} - -template< performance_data::t_creation_type CreationType, typename Index > -uint32_t performance_impl::delete_old_objects( const Index& old_idx, const account_name_type& start_account, uint32_t max_size, performance_data& pd ) const -{ - auto it_l = old_idx.lower_bound( start_account ); - auto it_u = old_idx.upper_bound( start_account ); - - if( it_l == it_u ) - return 0; - - uint32_t next_id = get_actual_id( *it_l ) + 1; - - auto r_end = old_idx.rend(); - decltype( r_end ) it( it_u ); - - bool is_init = true; - - while( it != r_end && it->account == start_account && next_id - get_actual_id( *it ) > max_size ) - { - auto old_itr = it; - ++it; - - remember_last< CreationType >( pd.s.is_empty, is_init, old_itr, pd ); - } - - if( !is_init ) - modify< CreationType >( *std::prev( it ), start_account, next_id, pd ); - - return next_id; -} - -performance::performance( database& _db ) - : my( new performance_impl( _db ) ) -{ - -} - -performance::~performance() -{ - -} - -template< performance_data::t_creation_type CreationType, typename Index > -uint32_t performance::delete_old_objects( const Index& old_idx, const account_name_type& start_account, uint32_t max_size, performance_data& pd ) const -{ - FC_ASSERT( my ); - return my->delete_old_objects< CreationType >( old_idx, start_account, max_size, pd ); -} - -using t_feed = decltype( ((database*)nullptr)->get_index< feed_index >().indices().get< by_feed >() ); -using t_blog = decltype( ((database*)nullptr)->get_index< blog_index >().indices().get< by_blog >() ); - -template uint32_t performance::delete_old_objects< performance_data::t_creation_type::full_feed >( const t_feed& old_idx, const account_name_type& start_account, uint32_t max_size, performance_data& pd ) const; -template uint32_t performance::delete_old_objects< performance_data::t_creation_type::part_feed >( const t_feed& old_idx, const account_name_type& start_account, uint32_t max_size, performance_data& pd ) const; -template uint32_t performance::delete_old_objects< performance_data::t_creation_type::full_blog >( const t_blog& old_idx, const account_name_type& start_account, uint32_t max_size, performance_data& pd ) const; - -} } } //steem::follow diff --git a/libraries/plugins/follow/include/steem/plugins/follow/follow_objects.hpp b/libraries/plugins/follow/include/steem/plugins/follow/follow_objects.hpp index 290c7f0b76..3c41e25312 100644 --- a/libraries/plugins/follow/include/steem/plugins/follow/follow_objects.hpp +++ b/libraries/plugins/follow/include/steem/plugins/follow/follow_objects.hpp @@ -7,10 +7,7 @@ namespace steem { namespace plugins { namespace follow { using namespace std; using namespace steem::chain; - -//using chainbase::t_vector; //Only when std::allocator is introduced. -template -using t_vector = chainbase::shared_vector< T >;//This will be removed, when std::allocator will be introduced. +using chainbase::shared_vector; #ifndef STEEM_FOLLOW_SPACE_ID #define STEEM_FOLLOW_SPACE_ID 8 @@ -57,13 +54,11 @@ typedef oid< follow_object > follow_id_type; class feed_object : public object< feed_object_type, feed_object > { public: - typedef t_vector t_reblogged_by_container; - feed_object() = delete; template< typename Constructor, typename Allocator > feed_object( Constructor&& c, allocator< Allocator > a ) - :reblogged_by( a ) + :reblogged_by( a.get_segment_manager() ) { c( *this ); } @@ -71,10 +66,11 @@ class feed_object : public object< feed_object_type, feed_object > id_type id; account_name_type account; - t_reblogged_by_container reblogged_by; + shared_vector reblogged_by; account_name_type first_reblogged_by; time_point_sec first_reblogged_on; comment_id_type comment; + uint32_t reblogs; uint32_t account_feed_id = 0; }; @@ -213,6 +209,7 @@ typedef chainbase::shared_multi_index_container< > blog_author_stats_index; struct by_feed; +struct by_old_feed; struct by_account; struct by_comment; @@ -227,6 +224,20 @@ typedef multi_index_container< >, composite_key_compare< std::less< account_name_type >, std::greater< uint32_t > > >, + ordered_unique< tag< by_old_feed >, + composite_key< feed_object, + member< feed_object, account_name_type, &feed_object::account >, + member< feed_object, uint32_t, &feed_object::account_feed_id > + >, + composite_key_compare< std::less< account_name_type >, std::less< uint32_t > > + >, + ordered_unique< tag< by_account >, + composite_key< feed_object, + member< feed_object, account_name_type, &feed_object::account >, + member< feed_object, feed_id_type, &feed_object::id > + >, + composite_key_compare< std::less< account_name_type >, std::less< feed_id_type > > + >, ordered_unique< tag< by_comment >, composite_key< feed_object, member< feed_object, comment_id_type, &feed_object::comment >, @@ -239,6 +250,7 @@ typedef multi_index_container< > feed_index; struct by_blog; +struct by_old_blog; typedef multi_index_container< blog_object, @@ -251,6 +263,13 @@ typedef multi_index_container< >, composite_key_compare< std::less< account_name_type >, std::greater< uint32_t > > >, + ordered_unique< tag< by_old_blog >, + composite_key< blog_object, + member< blog_object, account_name_type, &blog_object::account >, + member< blog_object, uint32_t, &blog_object::blog_feed_id > + >, + composite_key_compare< std::less< account_name_type >, std::less< uint32_t > > + >, ordered_unique< tag< by_comment >, composite_key< blog_object, member< blog_object, comment_id_type, &blog_object::comment >, @@ -262,10 +281,19 @@ typedef multi_index_container< allocator< blog_object > > blog_index; +struct by_reputation; + typedef multi_index_container< reputation_object, indexed_by< ordered_unique< tag< by_id >, member< reputation_object, reputation_id_type, &reputation_object::id > >, + ordered_unique< tag< by_reputation >, + composite_key< reputation_object, + member< reputation_object, share_type, &reputation_object::reputation >, + member< reputation_object, account_name_type, &reputation_object::account > + >, + composite_key_compare< std::greater< share_type >, std::less< account_name_type > > + >, ordered_unique< tag< by_account >, member< reputation_object, account_name_type, &reputation_object::account > > >, allocator< reputation_object > @@ -279,7 +307,21 @@ typedef multi_index_container< follow_count_object, indexed_by< ordered_unique< tag< by_id >, member< follow_count_object, follow_count_id_type, &follow_count_object::id > >, - ordered_unique< tag< by_account >, member< follow_count_object, account_name_type, &follow_count_object::account > > + ordered_unique< tag< by_account >, member< follow_count_object, account_name_type, &follow_count_object::account > >, + ordered_unique< tag< by_followers >, + composite_key< follow_count_object, + member< follow_count_object, uint32_t, &follow_count_object::follower_count >, + member< follow_count_object, follow_count_id_type, &follow_count_object::id > + >, + composite_key_compare< std::greater< uint32_t >, std::less< follow_count_id_type > > + >, + ordered_unique< tag< by_following >, + composite_key< follow_count_object, + member< follow_count_object, uint32_t, &follow_count_object::following_count >, + member< follow_count_object, follow_count_id_type, &follow_count_object::id > + >, + composite_key_compare< std::greater< uint32_t >, std::less< follow_count_id_type > > + > >, allocator< follow_count_object > > follow_count_index; @@ -291,7 +333,7 @@ FC_REFLECT_ENUM( steem::plugins::follow::follow_type, (undefined)(blog)(ignore) FC_REFLECT( steem::plugins::follow::follow_object, (id)(follower)(following)(what) ) CHAINBASE_SET_INDEX_TYPE( steem::plugins::follow::follow_object, steem::plugins::follow::follow_index ) -FC_REFLECT( steem::plugins::follow::feed_object, (id)(account)(first_reblogged_by)(first_reblogged_on)(reblogged_by)(comment)(account_feed_id) ) +FC_REFLECT( steem::plugins::follow::feed_object, (id)(account)(first_reblogged_by)(first_reblogged_on)(reblogged_by)(comment)(reblogs)(account_feed_id) ) CHAINBASE_SET_INDEX_TYPE( steem::plugins::follow::feed_object, steem::plugins::follow::feed_index ) FC_REFLECT( steem::plugins::follow::blog_object, (id)(account)(comment)(reblogged_on)(blog_feed_id) ) diff --git a/libraries/plugins/follow/include/steem/plugins/follow/inc_performance.hpp b/libraries/plugins/follow/include/steem/plugins/follow/inc_performance.hpp deleted file mode 100644 index f463410361..0000000000 --- a/libraries/plugins/follow/include/steem/plugins/follow/inc_performance.hpp +++ /dev/null @@ -1,133 +0,0 @@ -#pragma once - -#include -#include -#include - -namespace steem { namespace plugins{ namespace follow { - -using namespace steem::chain; -using steem::chain::database; - -using steem::protocol::account_name_type; - -class performance_impl; - -class dumper -{ - private: - - std::ofstream f; - - static std::unique_ptr< dumper > self; - - dumper() : -#if ENABLE_STD_ALLOCATOR == 1 - f( "std_dumped_objects.txt" ) -#else - f( "bip_dumped_objects.txt" ) -#endif - { - } - - public: - - ~dumper() - { - f.flush(); - f.close(); - } - - static std::unique_ptr< dumper >& instance() - { - if( !self ) - self = std::unique_ptr< dumper >( new dumper() ); - - return self; - } - - template< typename T, typename T2 > - void dump( const char* message, const T& data, const T2& data2 ) - { - static uint64_t counter = 0; - f< my; - - public: - - performance( database& _db ); - ~performance(); - - template< performance_data::t_creation_type CreationType, typename Index > - uint32_t delete_old_objects( const Index& old_idx, const account_name_type& start_account, uint32_t max_size, performance_data& pd ) const; - - template< typename T, typename T2 > - static void dump( const char* message, const T& data, const T2& data2 ) - { - dumper::instance()->dump( message, data, data2 ); - } -}; - -} } } //steem::follow