From 4be7bb55cc09e522688295274e3052f8d2ff7c61 Mon Sep 17 00:00:00 2001 From: maslenitsa93 Date: Thu, 21 Mar 2019 19:15:15 +0300 Subject: [PATCH 1/6] Add setting to view slow API requests #1222 --- .../include/golos/plugins/json_rpc/plugin.hpp | 3 +- plugins/json_rpc/plugin.cpp | 44 ++++++++++++++++--- tests/plugin_tests/json_rpc.cpp | 15 +++++-- 3 files changed, 50 insertions(+), 12 deletions(-) diff --git a/plugins/json_rpc/include/golos/plugins/json_rpc/plugin.hpp b/plugins/json_rpc/include/golos/plugins/json_rpc/plugin.hpp index 994be50c5c..14b6ed1c80 100644 --- a/plugins/json_rpc/include/golos/plugins/json_rpc/plugin.hpp +++ b/plugins/json_rpc/include/golos/plugins/json_rpc/plugin.hpp @@ -96,8 +96,7 @@ namespace golos { APPBASE_PLUGIN_REQUIRES(); void set_program_options(boost::program_options::options_description &, - boost::program_options::options_description &) override { - } + boost::program_options::options_description &); static const std::string &name() { static std::string name = STEEM_JSON_RPC_PLUGIN_NAME; diff --git a/plugins/json_rpc/plugin.cpp b/plugins/json_rpc/plugin.cpp index 5f94414940..d06f1f2c70 100644 --- a/plugins/json_rpc/plugin.cpp +++ b/plugins/json_rpc/plugin.cpp @@ -13,6 +13,9 @@ namespace golos { namespace plugins { namespace json_rpc { + + namespace bpo = boost::program_options; + struct json_rpc_error { json_rpc_error() : code(0) { } @@ -273,24 +276,41 @@ namespace golos { } struct dump_rpc_time { - dump_rpc_time(const fc::variant& data) - : data_(data) { + dump_rpc_time(const fc::variant& data, uint64_t log_rpc_calls_slower_msec) + : data_(data), log_rpc_calls_slower_msec_(log_rpc_calls_slower_msec) { dlog("data: ${data}", ("data", fc::json::to_string(data_))); } ~dump_rpc_time() { + auto msecs = (fc::time_point::now() - start_).count() / 1000; + if (error_.empty()) { dlog( - "elapsed: ${time} sec, data: ${data}", + "elapsed: ${time} msec, data: ${data}", ("data", fc::json::to_string(data_)) - ("time", double((fc::time_point::now() - start_).count()) / 1000000.0)); + ("time", msecs)); } else { dlog( - "elapsed: ${time} sec, error: '${error}', data: ${data}", + "elapsed: ${time} msec, error: '${error}', data: ${data}", ("data", fc::json::to_string(data_)) ("error", error_) - ("time", double((fc::time_point::now() - start_).count()) / 1000000.0)); + ("time", msecs)); + } + + if (uint64_t(msecs) > log_rpc_calls_slower_msec_) { + if (error_.empty()) { + wlog( + "Too slow RPC call: ${time} msec, data: ${data}", + ("data", fc::json::to_string(data_)) + ("time", msecs)); + } else { + wlog( + "Too slow RPC call: ${time} msec, error: '${error}', data: ${data}", + ("data", fc::json::to_string(data_)) + ("error", error_) + ("time", msecs)); + } } } @@ -302,10 +322,11 @@ namespace golos { fc::time_point start_ = fc::time_point::now(); std::string error_; const fc::variant& data_; + uint64_t log_rpc_calls_slower_msec_; }; void rpc(const fc::variant& data, msg_pack& msg) { - dump_rpc_time dump(data); + dump_rpc_time dump(data, _log_rpc_calls_slower_msec); try { rpc_jsonrpc(data, msg); @@ -406,6 +427,7 @@ namespace golos { map _registered_apis; vector _methods; map > _method_sigs; + uint64_t _log_rpc_calls_slower_msec = UINT64_MAX; private: // This is a reindex which allows to get parent plugin by method // unordered_map[method] -> plugin @@ -422,10 +444,18 @@ namespace golos { plugin::~plugin() { } + void plugin::set_program_options(bpo::options_description& cli, bpo::options_description& cfg) { + cfg.add_options() ( + "log-rpc-calls-slower-msec", bpo::value()->default_value(UINT64_MAX), + "Maximal milliseconds of RPC call or dump it as too slow. If not set, do not dump" + ); + } + void plugin::plugin_initialize(const boost::program_options::variables_map &options) { ilog("json_rpc plugin: plugin_initialize() begin"); pimpl = std::make_unique(); pimpl->initialize(); + pimpl->_log_rpc_calls_slower_msec = options.at("log-rpc-calls-slower-msec").as(); ilog("json_rpc plugin: plugin_initialize() end"); } diff --git a/tests/plugin_tests/json_rpc.cpp b/tests/plugin_tests/json_rpc.cpp index 517fe21618..e5a6c46274 100644 --- a/tests/plugin_tests/json_rpc.cpp +++ b/tests/plugin_tests/json_rpc.cpp @@ -117,9 +117,18 @@ BOOST_FIXTURE_TEST_SUITE(json_rpc, database_fixture) auto &rpc_plugin = appbase::app().register_plugin(); auto &testing_api = appbase::app().register_plugin(); - boost::program_options::variables_map options; - rpc_plugin.plugin_initialize(options); - testing_api.plugin_initialize(options); + { + boost::program_options::options_description desc; + rpc_plugin.set_program_options(desc, desc); + + boost::program_options::variables_map options; + boost::program_options::store(parse_command_line(0, (char**)NULL, desc), options); + rpc_plugin.plugin_initialize(options); + } + { + boost::program_options::variables_map options; + testing_api.plugin_initialize(options); + } open_database(); From 089c930530886f9eaebe4a5acf2662c6aefccfe4 Mon Sep 17 00:00:00 2001 From: zxcat <550974+zxcat@users.noreply.github.com> Date: Fri, 22 Feb 2019 18:17:50 +0300 Subject: [PATCH 2/6] Base state serializer #1133 note: removed duplicate `proposal_object` declaration --- .../golos/chain/steem_object_types.hpp | 1 - plugins/chain/CMakeLists.txt | 1 + .../include/golos/plugins/chain/plugin.hpp | 2 + plugins/chain/plugin.cpp | 26 ++- plugins/chain/serialize_state.cpp | 199 ++++++++++++++++++ 5 files changed, 227 insertions(+), 2 deletions(-) create mode 100644 plugins/chain/serialize_state.cpp diff --git a/libraries/chain/include/golos/chain/steem_object_types.hpp b/libraries/chain/include/golos/chain/steem_object_types.hpp index 9e7aa6a0f4..b8a64dc8ef 100644 --- a/libraries/chain/include/golos/chain/steem_object_types.hpp +++ b/libraries/chain/include/golos/chain/steem_object_types.hpp @@ -102,7 +102,6 @@ namespace golos { namespace chain { class vesting_delegation_object; class vesting_delegation_expiration_object; class account_metadata_object; - class proposal_object; typedef object_id dynamic_global_property_id_type; typedef object_id account_id_type; diff --git a/plugins/chain/CMakeLists.txt b/plugins/chain/CMakeLists.txt index 373382ab09..248aab966d 100644 --- a/plugins/chain/CMakeLists.txt +++ b/plugins/chain/CMakeLists.txt @@ -5,6 +5,7 @@ list(APPEND CURRENT_TARGET_HEADERS list(APPEND CURRENT_TARGET_SOURCES plugin.cpp + serialize_state.cpp ) if(BUILD_SHARED_LIBRARIES) diff --git a/plugins/chain/include/golos/plugins/chain/plugin.hpp b/plugins/chain/include/golos/plugins/chain/plugin.hpp index 357c645957..bed3714238 100644 --- a/plugins/chain/include/golos/plugins/chain/plugin.hpp +++ b/plugins/chain/include/golos/plugins/chain/plugin.hpp @@ -93,5 +93,7 @@ namespace golos { namespace plugins { namespace chain { friend struct golos::chain::database_fixture; // need to set skip_startup field bool skip_startup = false; + + void serialize_state(const boost::filesystem::path& output); }; } } } // golos::plugins::chain diff --git a/plugins/chain/plugin.cpp b/plugins/chain/plugin.cpp index 5c19753846..0ac4b89097 100644 --- a/plugins/chain/plugin.cpp +++ b/plugins/chain/plugin.cpp @@ -28,6 +28,10 @@ namespace golos { namespace plugins { namespace chain { bool readonly = false; bool check_locks = false; bool validate_invariants = false; + + bool serialize_state = false; + bfs::path serialize_state_path; + uint32_t flush_interval = 0; flat_map loaded_checkpoints; @@ -277,6 +281,10 @@ namespace golos { namespace plugins { namespace chain { ) ( "validate-database-invariants", bpo::bool_switch()->default_value(false), "Validate all supply invariants check out" + ) ( + "serialize-state", bpo::value(), + "The location of the file to serialize state to (abs path or relative to application data dir). " + "If set then app will exit after serialization." ); } @@ -331,6 +339,18 @@ namespace golos { namespace plugins { namespace chain { my->resync = options.at("resync-blockchain").as(); my->check_locks = options.at("check-locks").as(); my->validate_invariants = options.at("validate-database-invariants").as(); + + bool serialize = options.count("serialize-state") > 0; + if (serialize) { + auto s = options.at("serialize-state").as(); + serialize = s.size() > 0; + if (serialize) { + auto p = bfs::path(s); + my->serialize_state_path = p.is_relative() ? appbase::app().data_dir() / p : p; + } + } + my->serialize_state = serialize; + if (options.count("flush-state-interval")) { my->flush_interval = options.at("flush-state-interval").as(); } else { @@ -413,7 +433,11 @@ namespace golos { namespace plugins { namespace chain { auto head_block_log = my->db.get_block_log().head(); my->replay |= head_block_log && my->db.revision() != head_block_log->block_num(); - if (my->replay) { + if (my->serialize_state) { + serialize_state(my->serialize_state_path); + std::exit(0); // TODO Migrate to appbase::app().quit() + return; + } else if (my->replay) { my->replay_db(data_dir, my->force_replay); } } catch (const golos::chain::database_revision_exception&) { diff --git a/plugins/chain/serialize_state.cpp b/plugins/chain/serialize_state.cpp new file mode 100644 index 0000000000..077aeb15d2 --- /dev/null +++ b/plugins/chain/serialize_state.cpp @@ -0,0 +1,199 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +// #include + +#include +#include + + +namespace fc { + +template +inline datastream& operator<<(datastream& s, const chainbase::object_id& id) { + s.write((const char*)&id._id, sizeof(id._id)); + return s; +} +template +inline datastream& operator<<(datastream& s, const chainbase::shared_string& x) { + std::string t(x.data(), x.size()); + fc::raw::pack(s, t); + return s; +} + +} // fc + +namespace golos { namespace plugins { namespace chain { + + +namespace bfs = boost::filesystem; +using namespace golos::chain; + + +class ofstream_sha256: public bfs::ofstream { +public: + ofstream_sha256(const bfs::path& p): bfs::ofstream(p, std::ios_base::binary) { + bfs::ofstream::exceptions(std::ofstream::failbit | std::ofstream::badbit); + } + ~ofstream_sha256() { + } + + template + void write(const T& x) { + write((const char*)&x, sizeof(T)); + } + void write(const char* p, uint32_t l) { + _e.write(p, l); + bfs::ofstream::write(p, l); + } + fc::sha256 hash() { + return _e.result(); + } + +private: + fc::sha256::encoder _e; +}; + + +struct state_header { + char magic[12] = "Golos\astatE"; + uint32_t tables_count; +}; +struct table_header { + uint32_t type_id; + uint32_t records_count; +}; + +template +void serialize_table(const database& db, ofstream_sha256& out) { + auto start = fc::time_point::now(); + size_t n = 0, l = 0; + uint32_t min = -1, max = 0; + + const auto& generic = db.get_index(); + const auto& indices = generic.indicies(); + table_header hdr({chainbase::generic_index::value_type::type_id, static_cast(indices.size())}); + wlog("Saving ${name}, ${n} record(s), type: ${t}", + ("name", generic.name())("n", hdr.records_count)("t", hdr.type_id)); + out.write(hdr); + + const auto& idx = indices.template get(); + auto itr = idx.begin(); + auto etr = idx.end(); + for (; itr != etr; itr++) { + auto& item = *itr; + auto data = fc::raw::pack(item); + auto sz = data.size(); + if (sz < min) min = sz; + if (sz > max) max = sz; + l += sz; + out.write(data.data(), sz); + n++; + } + auto end = fc::time_point::now(); + ilog(" done, ${n} record(s) ${min}-${max} bytes each (${s.1} avg, ${l} total) saved in ${t} sec", + ("n", n)("min", min)("max", max)("l", l)("s", double(l)/n) + ("t", double((end - start).count()) / 1000000.0)); +} + + +void plugin::serialize_state(const bfs::path& output) { + // can't throw here, because if will be false-detected as db opening error, which can kill state + try { + ofstream_sha256 out(output); + auto start = fc::time_point::now(); + wlog("---------------------------------------------------------------------------"); + wlog("Serializing state to ${dst}", ("dst",output.string())); + auto& db_ = db(); + auto hdr = state_header{}; + hdr.tables_count = db_.index_list_size(); + out.write(hdr); + + for (auto i = db_.index_list_begin(), e = db_.index_list_end(); e != i; ++i) { + auto idx = *i; + ilog("index `${i}` (rev:${r}, type:${t}) contains ${l} records", + ("i",idx->name())("l",idx->size())("r",idx->revision())("t",idx->type_id())); + } + ilog("---------------------------------------------------------------------------"); + +#define STORE(T) serialize_table(db_, out); + STORE(dynamic_global_property_index); + STORE(account_index); + STORE(account_authority_index); + STORE(account_bandwidth_index); + STORE(witness_index); + STORE(transaction_index); + STORE(block_summary_index); + STORE(witness_schedule_index); + STORE(comment_index); + STORE(comment_vote_index); + STORE(witness_vote_index); + STORE(limit_order_index); + // STORE(feed_history_index); + STORE(convert_request_index); + STORE(liquidity_reward_balance_index); + // STORE(hardfork_property_index); + STORE(withdraw_vesting_route_index); + STORE(owner_authority_history_index); + STORE(account_recovery_request_index); + STORE(change_recovery_account_request_index); + STORE(escrow_index); + STORE(savings_withdraw_index); + STORE(decline_voting_rights_request_index); + STORE(vesting_delegation_index); + STORE(vesting_delegation_expiration_index); + STORE(account_metadata_index); + // STORE(proposal_index); + // STORE(required_approval_index); + + // STORE(golos::plugins::follow::follow_index); + // STORE(golos::plugins::follow::feed_index); + // STORE(golos::plugins::follow::blog_index); + // STORE(golos::plugins::follow::reputation_index); + // STORE(golos::plugins::follow::follow_count_index); + // STORE(golos::plugins::follow::blog_author_stats_index); + // STORE(golos::plugins::social_network::comment_content_index); + // STORE(golos::plugins::social_network::comment_last_update_index); + // STORE(golos::plugins::social_network::comment_reward_index); +#undef STORE + + auto end = fc::time_point::now(); + wlog("Done in ${t} sec.", ("t", double((end - start).count()) / 1000000.0)); + wlog("SHA256 hash: ${h}", ("h", out.hash().str())); + out.close(); + + } catch (const boost::exception& e) { + std::cerr << boost::diagnostic_information(e) << "\n"; + } catch (const fc::exception& e) { + std::cerr << e.to_detail_string() << "\n"; + } catch (const std::exception& e) { + std::cerr << e.what() << "\n"; + } catch (...) { + std::cerr << "unknown exception\n"; + } +} + +}}} // golos::plugins::chain + +// missing reflections +FC_REFLECT((golos::chain::comment_object), + (id)(parent_author)(parent_permlink)(author)(permlink)(created)(last_payout)(depth)(children) + (children_rshares2)(net_rshares)(abs_rshares)(vote_rshares)(children_abs_rshares)(cashout_time)(max_cashout_time) + (reward_weight)(net_votes)(total_votes)(root_comment)(mode) + (curation_reward_curve)(auction_window_reward_destination)(auction_window_size)(max_accepted_payout) + (percent_steem_dollars)(allow_replies)(allow_votes)(allow_curation_rewards)(curation_rewards_percent) + (beneficiaries)); + +FC_REFLECT((golos::chain::delegator_vote_interest_rate), (account)(interest_rate)(payout_strategy)); + +FC_REFLECT((golos::chain::comment_vote_object), + (id)(voter)(comment)(orig_rshares)(rshares)(vote_percent)(auction_time)(last_update)(num_changes) + (delegator_vote_interest_rates)) + +FC_REFLECT((golos::chain::witness_vote_object), (id)(witness)(account)); From a1913d4712664d63b1e0d781f1af4a4ef11a0b98 Mon Sep 17 00:00:00 2001 From: zxcat <550974+zxcat@users.noreply.github.com> Date: Fri, 1 Mar 2019 10:51:38 +0300 Subject: [PATCH 3/6] Use account & permlink maps in state serializer #1133 + store accounts index first + use variable length idx for map lookup --- plugins/chain/serialize_state.cpp | 111 +++++++++++++++++++++++++++--- plugins/chain/serialize_state.hpp | 25 +++++++ 2 files changed, 127 insertions(+), 9 deletions(-) create mode 100644 plugins/chain/serialize_state.hpp diff --git a/plugins/chain/serialize_state.cpp b/plugins/chain/serialize_state.cpp index 077aeb15d2..87904d2a28 100644 --- a/plugins/chain/serialize_state.cpp +++ b/plugins/chain/serialize_state.cpp @@ -1,3 +1,4 @@ +#include "serialize_state.hpp" #include #include @@ -12,6 +13,70 @@ #include #include +#define ID_T unsigned_int +// #define ID_T uint32_t + +namespace golos { namespace custom_pack { + +enum str_type { + permlink, + meta, + other, + _size +}; +struct str_info { + std::vector items; + std::map ids; // to get id fast +}; + +static str_type _current_str_type = other; +static str_info _stats[str_type::_size]; +static str_info _accs_stats; + +uint32_t put_item(str_info& info, const std::string& s) { + uint32_t id = info.items.size(); + if (info.ids.count(s)) { + id = info.ids[s]; + } else { + info.ids[s] = id; + info.items.push_back(s); + } + return id; +} +uint32_t put_str(const std::string& s) { + return put_item(_stats[_current_str_type], s); +} +uint32_t put_acc(const std::string& a) { + return put_item(_accs_stats, a); +} + +}} + +namespace fc { namespace raw { + +template +void pack(S& s, const golos::chain::account_name_type& a) { + auto id = golos::custom_pack::put_acc(std::string(a)); + fc::raw::pack(s, ID_T(id)); // variable-length +} +template +void pack(S& s, const golos::chain::shared_authority& a) { + fc::raw::pack(s, a.weight_threshold); + // account_auths is not properly serialized (string instead of account_name_type), so pack it manually + fc::raw::pack(s, unsigned_int((uint32_t)a.account_auths.size())); + for (const auto& acc : a.account_auths) { + fc::raw::pack(s, golos::chain::account_name_type(acc.first)); + fc::raw::pack(s, acc.second); + } + fc::raw::pack(s, a.key_auths); +} +template +void pack(S& s, const golos::protocol::beneficiary_route_type& b) { + fc::raw::pack(s, golos::chain::account_name_type(b.account)); + fc::raw::pack(s, b.weight); +} + +}} // fc::raw namespace fc { @@ -22,8 +87,13 @@ inline datastream& operator<<(datastream& s, const chainbase::object_id } template inline datastream& operator<<(datastream& s, const chainbase::shared_string& x) { - std::string t(x.data(), x.size()); - fc::raw::pack(s, t); + const auto t = golos::chain::to_string(x); + if (golos::custom_pack::_current_str_type == golos::custom_pack::other) { + fc::raw::pack(s, t); + } else { + auto id = golos::custom_pack::put_str(t); + fc::raw::pack(s, ID_T(id)); + } return s; } @@ -60,10 +130,9 @@ class ofstream_sha256: public bfs::ofstream { fc::sha256::encoder _e; }; - struct state_header { char magic[12] = "Golos\astatE"; - uint32_t tables_count; + uint32_t version; }; struct table_header { uint32_t type_id; @@ -97,7 +166,7 @@ void serialize_table(const database& db, ofstream_sha256& out) { n++; } auto end = fc::time_point::now(); - ilog(" done, ${n} record(s) ${min}-${max} bytes each (${s.1} avg, ${l} total) saved in ${t} sec", + ilog(" done, ${n} record(s) ${min}-${max} bytes each (${s} avg, ${l} total) saved in ${t} sec", ("n", n)("min", min)("max", max)("l", l)("s", double(l)/n) ("t", double((end - start).count()) / 1000000.0)); } @@ -112,7 +181,7 @@ void plugin::serialize_state(const bfs::path& output) { wlog("Serializing state to ${dst}", ("dst",output.string())); auto& db_ = db(); auto hdr = state_header{}; - hdr.tables_count = db_.index_list_size(); + hdr.version = 1;//db_.index_list_size(); out.write(hdr); for (auto i = db_.index_list_begin(), e = db_.index_list_end(); e != i; ++i) { @@ -123,16 +192,18 @@ void plugin::serialize_state(const bfs::path& output) { ilog("---------------------------------------------------------------------------"); #define STORE(T) serialize_table(db_, out); - STORE(dynamic_global_property_index); STORE(account_index); STORE(account_authority_index); STORE(account_bandwidth_index); + STORE(dynamic_global_property_index); STORE(witness_index); - STORE(transaction_index); + //STORE(transaction_index); STORE(block_summary_index); STORE(witness_schedule_index); + custom_pack::_current_str_type = custom_pack::permlink; STORE(comment_index); STORE(comment_vote_index); + custom_pack::_current_str_type = custom_pack::other; STORE(witness_vote_index); STORE(limit_order_index); // STORE(feed_history_index); @@ -148,7 +219,9 @@ void plugin::serialize_state(const bfs::path& output) { STORE(decline_voting_rights_request_index); STORE(vesting_delegation_index); STORE(vesting_delegation_expiration_index); + // custom_pack::_current_str_type = custom_pack::meta; STORE(account_metadata_index); + // custom_pack::_current_str_type = custom_pack::other; // STORE(proposal_index); // STORE(required_approval_index); @@ -165,7 +238,27 @@ void plugin::serialize_state(const bfs::path& output) { auto end = fc::time_point::now(); wlog("Done in ${t} sec.", ("t", double((end - start).count()) / 1000000.0)); - wlog("SHA256 hash: ${h}", ("h", out.hash().str())); + wlog("Data SHA256 hash: ${h}", ("h", out.hash().str())); + out.close(); + + auto map_file = output; + map_file += ".map"; + ofstream_sha256 om(map_file); + auto store_map_table = [&](char type, const custom_pack::str_info& info) { + uint32_t l = info.ids.size(); + om.write(type); + om.write(l); + for (const auto& i: info.items) { + om.write(i.c_str(), i.size()); + om.write('\0'); + } + }; + + store_map_table('A', custom_pack::_accs_stats); + store_map_table('P', custom_pack::_stats[custom_pack::permlink]); + store_map_table('M', custom_pack::_stats[custom_pack::meta]); + + wlog("Map SHA256 hash: ${h}", ("h", om.hash().str())); out.close(); } catch (const boost::exception& e) { diff --git a/plugins/chain/serialize_state.hpp b/plugins/chain/serialize_state.hpp new file mode 100644 index 0000000000..16d0140e37 --- /dev/null +++ b/plugins/chain/serialize_state.hpp @@ -0,0 +1,25 @@ +#pragma once + +// this file must only be included from serialize_state.cpp +// required to declare some packers before so they become visible to compiler + +namespace fc { + class uint128_t; + template class fixed_string; +} +namespace golos { namespace chain { + using account_name_type = fc::fixed_string; + struct shared_authority; +}} +namespace golos { namespace protocol { + struct beneficiary_route_type; +}} + + +namespace fc { namespace raw { + +template void pack(S&, const golos::chain::account_name_type&); +template void pack(S&, const golos::chain::shared_authority&); +template void pack(S&, const golos::protocol::beneficiary_route_type&); + +}} // fc::raw From ef7858ab7a0d3f27eb51623c4f32671cb7e2aaa2 Mon Sep 17 00:00:00 2001 From: zxcat <550974+zxcat@users.noreply.github.com> Date: Fri, 1 Mar 2019 12:31:06 +0300 Subject: [PATCH 4/6] Serialize to state only consensus data of archived comments #1133 --- plugins/chain/serialize_state.cpp | 51 ++++++++++++++++++++++++++++--- plugins/chain/serialize_state.hpp | 2 ++ 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/plugins/chain/serialize_state.cpp b/plugins/chain/serialize_state.cpp index 87904d2a28..e004b2718a 100644 --- a/plugins/chain/serialize_state.cpp +++ b/plugins/chain/serialize_state.cpp @@ -76,6 +76,44 @@ void pack(S& s, const golos::protocol::beneficiary_route_type& b) { fc::raw::pack(s, b.weight); } +// store only consensus data for archived comments +template +void pack(S& s, const golos::chain::comment_object& c) { + fc::raw::pack(s, c.id); + fc::raw::pack(s, c.parent_author); + fc::raw::pack(s, c.parent_permlink); + fc::raw::pack(s, c.author); + fc::raw::pack(s, c.permlink); + fc::raw::pack(s, c.mode); + if (c.mode != golos::chain::comment_mode::archived) { + fc::raw::pack(s, c.created); + fc::raw::pack(s, c.last_payout); + fc::raw::pack(s, c.depth); + fc::raw::pack(s, c.children); + fc::raw::pack(s, c.children_rshares2); + fc::raw::pack(s, c.net_rshares); + fc::raw::pack(s, c.abs_rshares); + fc::raw::pack(s, c.vote_rshares); + fc::raw::pack(s, c.children_abs_rshares); + fc::raw::pack(s, c.cashout_time); + fc::raw::pack(s, c.max_cashout_time); + fc::raw::pack(s, c.reward_weight); + fc::raw::pack(s, c.net_votes); + fc::raw::pack(s, c.total_votes); + fc::raw::pack(s, c.root_comment); + fc::raw::pack(s, c.curation_reward_curve); + fc::raw::pack(s, c.auction_window_reward_destination); + fc::raw::pack(s, c.auction_window_size); + fc::raw::pack(s, c.max_accepted_payout); + fc::raw::pack(s, c.percent_steem_dollars); + fc::raw::pack(s, c.allow_replies); + fc::raw::pack(s, c.allow_votes); + fc::raw::pack(s, c.allow_curation_rewards); + fc::raw::pack(s, c.curation_rewards_percent); + fc::raw::pack(s, c.beneficiaries); + } +} + }} // fc::raw namespace fc { @@ -259,7 +297,7 @@ void plugin::serialize_state(const bfs::path& output) { store_map_table('M', custom_pack::_stats[custom_pack::meta]); wlog("Map SHA256 hash: ${h}", ("h", om.hash().str())); - out.close(); + om.close(); } catch (const boost::exception& e) { std::cerr << boost::diagnostic_information(e) << "\n"; @@ -274,15 +312,18 @@ void plugin::serialize_state(const bfs::path& output) { }}} // golos::plugins::chain -// missing reflections +// remove reflect of comment_object and serialize it manually +/* FC_REFLECT((golos::chain::comment_object), - (id)(parent_author)(parent_permlink)(author)(permlink)(created)(last_payout)(depth)(children) + (id)(parent_author)(parent_permlink)(author)(permlink)(mode) // consensus part + (created)(last_payout)(depth)(children) (children_rshares2)(net_rshares)(abs_rshares)(vote_rshares)(children_abs_rshares)(cashout_time)(max_cashout_time) - (reward_weight)(net_votes)(total_votes)(root_comment)(mode) + (reward_weight)(net_votes)(total_votes)(root_comment) (curation_reward_curve)(auction_window_reward_destination)(auction_window_size)(max_accepted_payout) (percent_steem_dollars)(allow_replies)(allow_votes)(allow_curation_rewards)(curation_rewards_percent) - (beneficiaries)); + (beneficiaries));*/ +// missing reflections FC_REFLECT((golos::chain::delegator_vote_interest_rate), (account)(interest_rate)(payout_strategy)); FC_REFLECT((golos::chain::comment_vote_object), diff --git a/plugins/chain/serialize_state.hpp b/plugins/chain/serialize_state.hpp index 16d0140e37..7a07b73392 100644 --- a/plugins/chain/serialize_state.hpp +++ b/plugins/chain/serialize_state.hpp @@ -10,6 +10,7 @@ namespace fc { namespace golos { namespace chain { using account_name_type = fc::fixed_string; struct shared_authority; + class comment_object; }} namespace golos { namespace protocol { struct beneficiary_route_type; @@ -18,6 +19,7 @@ namespace golos { namespace protocol { namespace fc { namespace raw { +template void pack(S&, const golos::chain::comment_object&); template void pack(S&, const golos::chain::account_name_type&); template void pack(S&, const golos::chain::shared_authority&); template void pack(S&, const golos::protocol::beneficiary_route_type&); From 688fc91c20f87a371ac8fef126fba4a9dfbf715f Mon Sep 17 00:00:00 2001 From: boostmultifruit Date: Fri, 12 Apr 2019 11:40:39 +0300 Subject: [PATCH 5/6] Operation dump plugin #1287 --- libraries/chain/database.cpp | 5 + .../include/golos/protocol/operations.hpp | 3 +- .../protocol/steem_virtual_operations.hpp | 17 ++ plugins/account_history/plugin.cpp | 4 + .../plugins/mongo_db/mongo_db_operations.hpp | 1 + .../golos/plugins/mongo_db/mongo_db_state.hpp | 1 + plugins/mongo_db/mongo_db_operations.cpp | 5 + plugins/mongo_db/mongo_db_state.cpp | 4 + plugins/operation_dump/CMakeLists.txt | 44 +++++ .../operation_dump_container.hpp | 42 ++++ .../operation_dump/operation_dump_plugin.hpp | 37 ++++ .../operation_dump/operation_dump_visitor.hpp | 184 ++++++++++++++++++ .../operation_dump/operation_dump_plugin.cpp | 135 +++++++++++++ programs/golosd/CMakeLists.txt | 1 + programs/golosd/main.cpp | 2 + share/golosd/config/config.ini | 2 +- share/golosd/config/config_debug.ini | 2 +- share/golosd/config/config_debug_mongo.ini | 2 +- share/golosd/config/config_mongo.ini | 2 +- 19 files changed, 488 insertions(+), 5 deletions(-) create mode 100644 plugins/operation_dump/CMakeLists.txt create mode 100644 plugins/operation_dump/include/golos/plugins/operation_dump/operation_dump_container.hpp create mode 100644 plugins/operation_dump/include/golos/plugins/operation_dump/operation_dump_plugin.hpp create mode 100644 plugins/operation_dump/include/golos/plugins/operation_dump/operation_dump_visitor.hpp create mode 100644 plugins/operation_dump/operation_dump_plugin.cpp diff --git a/libraries/chain/database.cpp b/libraries/chain/database.cpp index cb999b7fb0..37dd2702c3 100644 --- a/libraries/chain/database.cpp +++ b/libraries/chain/database.cpp @@ -2467,6 +2467,11 @@ namespace golos { namespace chain { modify(get_account(comment.author), [&](account_object &a) { a.posting_rewards += author_tokens; }); + + auto author_golos = asset(author_tokens, STEEM_SYMBOL); + auto benefactor_golos = asset(total_beneficiary, STEEM_SYMBOL); + auto curator_golos = asset(reward_tokens.to_uint64() - (author_tokens + total_beneficiary), STEEM_SYMBOL); + push_virtual_operation(total_comment_reward_operation(comment.author, to_string(comment.permlink), author_golos, benefactor_golos, curator_golos, comment.net_rshares.value)); } fc::uint128_t old_rshares2 = calculate_vshares(comment.net_rshares.value); diff --git a/libraries/protocol/include/golos/protocol/operations.hpp b/libraries/protocol/include/golos/protocol/operations.hpp index cb00d3e693..9adc311df3 100644 --- a/libraries/protocol/include/golos/protocol/operations.hpp +++ b/libraries/protocol/include/golos/protocol/operations.hpp @@ -87,7 +87,8 @@ namespace golos { namespace protocol { return_vesting_delegation_operation, producer_reward_operation, delegation_reward_operation, - auction_window_reward_operation + auction_window_reward_operation, + total_comment_reward_operation > operation; /*void operation_get_required_authorities( const operation& op, diff --git a/libraries/protocol/include/golos/protocol/steem_virtual_operations.hpp b/libraries/protocol/include/golos/protocol/steem_virtual_operations.hpp index 68901a2aaf..d4fc8f20c9 100644 --- a/libraries/protocol/include/golos/protocol/steem_virtual_operations.hpp +++ b/libraries/protocol/include/golos/protocol/steem_virtual_operations.hpp @@ -232,6 +232,22 @@ namespace golos { namespace protocol { account_name_type account; asset vesting_shares; }; + + struct total_comment_reward_operation : public virtual_operation { + total_comment_reward_operation() { + } + + total_comment_reward_operation(const account_name_type& a, const string &p, const asset& ar, const asset& br, const asset& cr, int64_t nr) + : author(a), permlink(p), author_reward(ar), benefactor_reward(br), curator_reward(cr), net_rshares(nr) { + } + + account_name_type author; + string permlink; + asset author_reward; + asset benefactor_reward; + asset curator_reward; + int64_t net_rshares; + }; } } //golos::protocol FC_REFLECT((golos::protocol::author_reward_operation), (author)(permlink)(sbd_payout)(steem_payout)(vesting_payout)) @@ -251,3 +267,4 @@ FC_REFLECT((golos::protocol::comment_benefactor_reward_operation), (benefactor)( FC_REFLECT((golos::protocol::return_vesting_delegation_operation), (account)(vesting_shares)) FC_REFLECT((golos::protocol::producer_reward_operation), (producer)(vesting_shares)) FC_REFLECT((golos::protocol::delegation_reward_operation), (delegator)(delegatee)(payout_strategy)(vesting_shares)) +FC_REFLECT((golos::protocol::total_comment_reward_operation), (author)(permlink)(author_reward)(benefactor_reward)(curator_reward)(net_rshares)) diff --git a/plugins/account_history/plugin.cpp b/plugins/account_history/plugin.cpp index b72d83bf9f..681775e39e 100644 --- a/plugins/account_history/plugin.cpp +++ b/plugins/account_history/plugin.cpp @@ -486,6 +486,10 @@ if (options.count(name)) { \ insert_receiver(op.account); } + void operator()(const total_comment_reward_operation& op) { + insert_dual(op.author); + } + // todo: proposal tx signers are receivers void operator()(const proposal_create_operation& op) { insert_dual(op.author); diff --git a/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_operations.hpp b/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_operations.hpp index 5ebabdf0d2..18723efa6a 100644 --- a/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_operations.hpp +++ b/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_operations.hpp @@ -86,6 +86,7 @@ namespace mongo_db { result_type operator()(const chain_properties_update_operation& op); result_type operator()(const delegation_reward_operation& op); result_type operator()(const auction_window_reward_operation& op); + result_type operator()(const total_comment_reward_operation& op); }; }}} // golos::plugins::mongo_db diff --git a/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_state.hpp b/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_state.hpp index 01c75fe43f..5133c2a29d 100644 --- a/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_state.hpp +++ b/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_state.hpp @@ -88,6 +88,7 @@ namespace mongo_db { result_type operator()(const chain_properties_update_operation& op); result_type operator()(const delegation_reward_operation& op); result_type operator()(const auction_window_reward_operation& op); + result_type operator()(const total_comment_reward_operation& op); void write_global_property_object(const dynamic_global_property_object& dgpo, bool history); diff --git a/plugins/mongo_db/mongo_db_operations.cpp b/plugins/mongo_db/mongo_db_operations.cpp index 4cbb08030d..d7261271b3 100644 --- a/plugins/mongo_db/mongo_db_operations.cpp +++ b/plugins/mongo_db/mongo_db_operations.cpp @@ -768,4 +768,9 @@ namespace mongo_db { return body; } + auto operation_writer::operator()(const total_comment_reward_operation& op) -> result_type { + result_type body; + return body; + } + }}} diff --git a/plugins/mongo_db/mongo_db_state.cpp b/plugins/mongo_db/mongo_db_state.cpp index 681650e3e1..52ff5844d6 100644 --- a/plugins/mongo_db/mongo_db_state.cpp +++ b/plugins/mongo_db/mongo_db_state.cpp @@ -1927,4 +1927,8 @@ namespace mongo_db { } + auto state_writer::operator()(const total_comment_reward_operation& op) -> result_type { + + } + }}} \ No newline at end of file diff --git a/plugins/operation_dump/CMakeLists.txt b/plugins/operation_dump/CMakeLists.txt new file mode 100644 index 0000000000..398d552381 --- /dev/null +++ b/plugins/operation_dump/CMakeLists.txt @@ -0,0 +1,44 @@ +set(CURRENT_TARGET operation_dump) + +list(APPEND CURRENT_TARGET_HEADERS + include/golos/plugins/operation_dump/operation_dump_plugin.hpp + include/golos/plugins/operation_dump/operation_dump_container.hpp +) + +list(APPEND CURRENT_TARGET_SOURCES + operation_dump_plugin.cpp +) + +if(BUILD_SHARED_LIBRARIES) + add_library(golos_${CURRENT_TARGET} SHARED + ${CURRENT_TARGET_HEADERS} + ${CURRENT_TARGET_SOURCES} + ) +else() + add_library(golos_${CURRENT_TARGET} STATIC + ${CURRENT_TARGET_HEADERS} + ${CURRENT_TARGET_SOURCES} + ) +endif() + +add_library(golos::${CURRENT_TARGET} ALIAS golos_${CURRENT_TARGET}) +set_property(TARGET golos_${CURRENT_TARGET} PROPERTY EXPORT_NAME ${CURRENT_TARGET}) + +target_link_libraries( + golos_${CURRENT_TARGET} + golos::chain_plugin + golos::follow + golos::tags + appbase +) + +target_include_directories(golos_${CURRENT_TARGET} + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include") + +install(TARGETS + golos_${CURRENT_TARGET} + + RUNTIME DESTINATION bin + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib +) diff --git a/plugins/operation_dump/include/golos/plugins/operation_dump/operation_dump_container.hpp b/plugins/operation_dump/include/golos/plugins/operation_dump/operation_dump_container.hpp new file mode 100644 index 0000000000..c89a6d7fce --- /dev/null +++ b/plugins/operation_dump/include/golos/plugins/operation_dump/operation_dump_container.hpp @@ -0,0 +1,42 @@ +#pragma once + +#include + +namespace golos { namespace plugins { namespace operation_dump { + +namespace bfs = boost::filesystem; + +// Structure size can differ - uses sizeof +struct dump_header { + char magic[13] = "Golos\adumpOP"; + uint32_t version = 1; +}; + +using operation_number = std::pair; + +class dump_buffer : public std::stringstream { +public: + dump_buffer() { + } + + using std::stringstream::write; + + void write(const operation_number& op_num) { + fc::raw::pack(*this, op_num); + } +}; + +using dump_buffers = std::map; + +class dump_file : public bfs::ofstream { +public: + dump_file(const bfs::path& p): bfs::ofstream(p, std::ios_base::binary | std::ios_base::app) { + bfs::ofstream::exceptions(std::ofstream::failbit | std::ofstream::badbit); + } + + void write(const dump_header& hdr) { + bfs::ofstream::write((const char*)&hdr, sizeof(dump_header)); + } +}; + +} } } // golos::plugins::operation_dump diff --git a/plugins/operation_dump/include/golos/plugins/operation_dump/operation_dump_plugin.hpp b/plugins/operation_dump/include/golos/plugins/operation_dump/operation_dump_plugin.hpp new file mode 100644 index 0000000000..51e0259a91 --- /dev/null +++ b/plugins/operation_dump/include/golos/plugins/operation_dump/operation_dump_plugin.hpp @@ -0,0 +1,37 @@ +#pragma once + +#include +#include +#include +#include + +namespace golos { namespace plugins { namespace operation_dump { + +namespace bpo = boost::program_options; +using namespace golos::chain; + +class operation_dump_plugin final : public appbase::plugin { +public: + APPBASE_PLUGIN_REQUIRES((chain::plugin)) + + operation_dump_plugin(); + + ~operation_dump_plugin(); + + void set_program_options(bpo::options_description& cli, bpo::options_description& cfg) override; + + void plugin_initialize(const bpo::variables_map& options) override; + + void plugin_startup() override; + + void plugin_shutdown() override; + + static const std::string& name(); + +private: + class operation_dump_plugin_impl; + + std::unique_ptr my; +}; + +} } } //golos::plugins::operation_dump diff --git a/plugins/operation_dump/include/golos/plugins/operation_dump/operation_dump_visitor.hpp b/plugins/operation_dump/include/golos/plugins/operation_dump/operation_dump_visitor.hpp new file mode 100644 index 0000000000..4c053a2363 --- /dev/null +++ b/plugins/operation_dump/include/golos/plugins/operation_dump/operation_dump_visitor.hpp @@ -0,0 +1,184 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace golos { namespace plugins { namespace operation_dump { + +using namespace golos::plugins::follow; + +#define COMMENT_ID(OP) hash_id(std::string(OP.author) + "/" + OP.permlink) + +#define TAGS_NUMBER 15 +#define TAG_MAX_LENGTH 512 + +class operation_dump_visitor { +public: + using result_type = void; + + dump_buffers& _buffers; + + const signed_block& _block; + uint16_t& _op_in_block; + + database& _db; + + operation_dump_visitor(dump_buffers& buffers, const signed_block& block, uint16_t& op_in_block, database& db) + : _buffers(buffers), _block(block), _op_in_block(op_in_block), _db(db) { + } + + uint64_t hash_id(const std::string& id) { + return fc::hash64(id.c_str(), id.length()); + } + + dump_buffer& write_op_header(const std::string& file_name, uint64_t op_related_id) { + auto& b = _buffers[file_name]; + b.write(operation_number(_block.block_num(), _op_in_block)); + fc::raw::pack(b, op_related_id); + return b; + } + + template + auto operator()(const T&) -> result_type { + } + + auto operator()(const transfer_operation& op) -> result_type { + auto& b = write_op_header("transfers", 0); + + fc::raw::pack(b, op); + } + + auto operator()(const comment_operation& op) -> result_type { + auto& b = write_op_header("comments", COMMENT_ID(op)); + + fc::raw::pack(b, op.parent_author); + fc::raw::pack(b, op.parent_permlink); + fc::raw::pack(b, op.author); + fc::raw::pack(b, op.permlink); + fc::raw::pack(b, op.title); + fc::raw::pack(b, op.body); + + auto meta = golos::plugins::tags::get_metadata(op.json_metadata, TAGS_NUMBER, TAG_MAX_LENGTH); + fc::raw::pack(b, meta); + } + + auto operator()(const delete_comment_operation& op) -> result_type { + if (_db.find_comment(op.author, op.permlink)) { + return; + } + + write_op_header("delete_comments", COMMENT_ID(op)); + } + + auto operator()(const comment_benefactor_reward_operation& op) -> result_type { + auto& b = write_op_header("benefactor_rewards", COMMENT_ID(op)); + + fc::raw::pack(b, op); + } + + auto operator()(const author_reward_operation& op) -> result_type { + auto& b = write_op_header("author_rewards", COMMENT_ID(op)); + + fc::raw::pack(b, op); + } + + auto operator()(const curation_reward_operation& op) -> result_type { + auto& b = write_op_header("curation_rewards", hash_id(std::string(op.comment_author) + "/" + op.comment_permlink)); + + fc::raw::pack(b, op); + } + + auto operator()(const auction_window_reward_operation& op) -> result_type { + auto& b = write_op_header("auction_window_rewards", hash_id(std::string(op.comment_author) + "/" + op.comment_permlink)); + + fc::raw::pack(b, op); + } + + auto operator()(const total_comment_reward_operation& op) -> result_type { + auto& b = write_op_header("total_comment_rewards", COMMENT_ID(op)); + + fc::raw::pack(b, op); + } + + auto operator()(const vote_operation& op) -> result_type { + auto& b = write_op_header("votes", COMMENT_ID(op)); + + fc::raw::pack(b, op); + + fc::raw::pack(b, _block.timestamp); + } + + // Not logs if operation failed in plugin, but logs if plugin not exists + auto operator()(const custom_json_operation& op) -> result_type { + if (op.id != "follow") { // follows, reblogs, delete_reblogs + return; + } + + std::vector fpops; + + auto v = fc::json::from_string(op.json); + try { + if (v.is_array() && v.size() > 0 && v.get_array()[0].is_array()) { + fc::from_variant(v, fpops); + } else { + fpops.emplace_back(); + fc::from_variant(v, fpops[0]); + } + } catch (...) { + // Normal cases failed, try this strange case from follow-plugin + try { + auto fop = v.as(); + fpops.emplace_back(fop); + } catch (...) { + } + } + + for (const follow_plugin_operation& fpop : fpops) { + fpop.visit(*this); + } + } + + auto operator()(const follow_operation& op) -> result_type { + auto& b = write_op_header("follows", hash_id(std::string(op.follower) + "/" + op.following)); + + fc::raw::pack(b, op.follower); + fc::raw::pack(b, op.following); + + uint16_t what = 0; + for (const auto& target : op.what) { + if (target == "blog") { + what |= 1 << blog; + } else if (target == "ignore") { + what |= 1 << ignore; + } + } + fc::raw::pack(b, what); + } + + auto operator()(const reblog_operation& op) -> result_type { + auto& b = write_op_header("reblogs", COMMENT_ID(op)); + + fc::raw::pack(b, op.account); + fc::raw::pack(b, op.author); + fc::raw::pack(b, op.permlink); + fc::raw::pack(b, op.title); // Usually empty, but no problems to dump + fc::raw::pack(b, op.body); + // Usually it is: + // {"app": "golos.io/0.1", "format": "text"} + // Not seems to be need + //fc::raw::pack(b, op.json_metadata); + + fc::raw::pack(b, _block.timestamp); + } + + auto operator()(const delete_reblog_operation& op) -> result_type { + auto& b = write_op_header("delete_reblogs", COMMENT_ID(op)); + + fc::raw::pack(b, op.account); + } +}; + +} } } // golos::plugins::operation_dump diff --git a/plugins/operation_dump/operation_dump_plugin.cpp b/plugins/operation_dump/operation_dump_plugin.cpp new file mode 100644 index 0000000000..7c40eb5fec --- /dev/null +++ b/plugins/operation_dump/operation_dump_plugin.cpp @@ -0,0 +1,135 @@ +#include +#include +#include +#include +#include + +namespace golos { namespace plugins { namespace operation_dump { + +namespace bfs = boost::filesystem; + +class operation_dump_plugin::operation_dump_plugin_impl final { +public: + operation_dump_plugin_impl() + : _db(appbase::app().get_plugin().db()) { + } + + ~operation_dump_plugin_impl() { + } + + void on_block(const signed_block& block) { + auto lib = _db.last_non_undoable_block_num(); + + for (auto block_num = start_block; block_num <= lib; ++block_num) { + auto block = _db.get_block_log().read_block_by_num(block_num); + if (!block) { + return; + } + + try { + uint16_t op_in_block = 0; + + operation_dump_visitor op_visitor(buffers, *block, op_in_block, _db); + + for (const auto& trx : block->transactions) { + for (const auto& op : trx.operations) { + op.visit(op_visitor); + ++op_in_block; + } + } + + for (const auto& op : virtual_ops[block_num]) { + op.visit(op_visitor); + ++op_in_block; + } + } catch (...) { + virtual_ops.erase(block_num); + start_block = block_num+1; + throw; + } + + virtual_ops.erase(block_num); + } + + start_block = lib+1; + + for (auto& it : buffers) { + bfs::create_directories(operation_dump_dir); + dump_file file(operation_dump_dir / it.first); + if (file.tellp() == 0) { + file.write(dump_header()); + } + file << it.second.rdbuf(); + } + buffers.clear(); + } + + void on_operation(const operation_notification& note) { + if (!is_virtual_operation(note.op)) { + return; + } + + virtual_ops[note.block].push_back(note.op); + + // remove ops if there were forks and rollbacks + auto itr = virtual_ops.find(note.block); + ++itr; + virtual_ops.erase(itr, virtual_ops.end()); + } + + database& _db; + + bfs::path operation_dump_dir; + + uint32_t start_block = 1; + + std::map> virtual_ops; + dump_buffers buffers; +}; + +operation_dump_plugin::operation_dump_plugin() = default; + +operation_dump_plugin::~operation_dump_plugin() = default; + +const std::string& operation_dump_plugin::name() { + static std::string name = "operation_dump"; + return name; +} + +void operation_dump_plugin::set_program_options(bpo::options_description& cli, bpo::options_description& cfg) { + cfg.add_options() ( + "operation-dump-dir", bpo::value()->default_value("operation_dump"), + "The location of the dir to dump operations to (abs path or relative to application data dir)." + ); +} + +void operation_dump_plugin::plugin_initialize(const bpo::variables_map& options) { + ilog("Initializing operation dump plugin"); + + my = std::make_unique(); + + auto odd = options.at("operation-dump-dir").as(); + if (odd.is_relative()) { + my->operation_dump_dir = appbase::app().data_dir() / odd; + } else { + my->operation_dump_dir = odd; + } + + my->_db.applied_block.connect([&](const signed_block& b) { + my->on_block(b); + }); + + my->_db.post_apply_operation.connect([&](const operation_notification& note) { + my->on_operation(note); + }); +} + +void operation_dump_plugin::plugin_startup() { + ilog("Starting up operation dump plugin"); +} + +void operation_dump_plugin::plugin_shutdown() { + ilog("Shutting down operation dump plugin"); +} + +} } } // golos::plugins::operation_dump diff --git a/programs/golosd/CMakeLists.txt b/programs/golosd/CMakeLists.txt index d02b81f5de..81895cbb1f 100644 --- a/programs/golosd/CMakeLists.txt +++ b/programs/golosd/CMakeLists.txt @@ -28,6 +28,7 @@ target_link_libraries( golos::social_network golos::tags golos::market_history + golos::operation_dump golos::operation_history golos::statsd golos::account_by_key diff --git a/programs/golosd/main.cpp b/programs/golosd/main.cpp index f855e59484..c6ea4a4e81 100644 --- a/programs/golosd/main.cpp +++ b/programs/golosd/main.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #ifdef MONGODB_PLUGIN_BUILT #include #endif @@ -81,6 +82,7 @@ namespace golos { appbase::app().register_plugin(); appbase::app().register_plugin(); appbase::app().register_plugin(); + appbase::app().register_plugin(); #ifdef MONGODB_PLUGIN_BUILT appbase::app().register_plugin(); #endif diff --git a/share/golosd/config/config.ini b/share/golosd/config/config.ini index f756fc3beb..f0c24505ac 100644 --- a/share/golosd/config/config.ini +++ b/share/golosd/config/config.ini @@ -66,7 +66,7 @@ inc-shared-file-size = 2G # and resizes. The optimal strategy is do checking of the free space, but not very often. block-num-check-free-size = 1000 # each 3000 seconds -plugin = chain p2p json_rpc webserver network_broadcast_api witness test_api database_api private_message follow social_network tags market_history account_by_key operation_history account_history account_notes statsd block_info raw_block witness_api +plugin = chain p2p json_rpc webserver network_broadcast_api witness test_api database_api private_message follow social_network tags market_history account_by_key operation_dump operation_history account_history account_notes statsd block_info raw_block witness_api # Remove votes before defined block, should increase performance clear-votes-before-block = 4294967295 # clear votes after each cashout diff --git a/share/golosd/config/config_debug.ini b/share/golosd/config/config_debug.ini index b3293e7df6..2a5d74e4f9 100644 --- a/share/golosd/config/config_debug.ini +++ b/share/golosd/config/config_debug.ini @@ -66,7 +66,7 @@ inc-shared-file-size = 100M # and resizes. The optimal strategy is do checking of the free space, but not very often. block-num-check-free-size = 10 # each 30 seconds -plugin = chain p2p json_rpc webserver network_broadcast_api witness test_api database_api private_message follow social_network tags market_history account_by_key account_history account_notes operation_history statsd block_info raw_block debug_node witness_api +plugin = chain p2p json_rpc webserver network_broadcast_api witness test_api database_api private_message follow social_network tags market_history account_by_key account_history account_notes operation_dump operation_history statsd block_info raw_block debug_node witness_api # Remove votes before defined block, should increase performance clear-votes-before-block = 0 # don't clear votes diff --git a/share/golosd/config/config_debug_mongo.ini b/share/golosd/config/config_debug_mongo.ini index b144cf19be..d459b6b73d 100644 --- a/share/golosd/config/config_debug_mongo.ini +++ b/share/golosd/config/config_debug_mongo.ini @@ -66,7 +66,7 @@ inc-shared-file-size = 100M # and resizes. The optimal strategy is do checking of the free space, but not very often. block-num-check-free-size = 10 # each 30 seconds -plugin = chain p2p json_rpc webserver network_broadcast_api witness test_api database_api private_message follow social_network tags market_history account_by_key account_history account_notes operation_history statsd block_info raw_block debug_node witness_api mongo_db +plugin = chain p2p json_rpc webserver network_broadcast_api witness test_api database_api private_message follow social_network tags market_history account_by_key account_history account_notes operation_dump operation_history statsd block_info raw_block debug_node witness_api mongo_db # For connect to mongodb which is running outside Docker (if golosd running inside) mongodb-uri = mongodb://172.17.0.1:27017/Golos diff --git a/share/golosd/config/config_mongo.ini b/share/golosd/config/config_mongo.ini index 9d477202e9..552ae7ee1e 100644 --- a/share/golosd/config/config_mongo.ini +++ b/share/golosd/config/config_mongo.ini @@ -66,7 +66,7 @@ inc-shared-file-size = 2G # and resizes. The optimal strategy is do checking of the free space, but not very often. block-num-check-free-size = 1000 # each 3000 seconds -plugin = chain p2p json_rpc webserver network_broadcast_api witness test_api database_api private_message follow social_network tags market_history account_by_key operation_history account_history account_notes statsd block_info raw_block witness_api mongo_db +plugin = chain p2p json_rpc webserver network_broadcast_api witness test_api database_api private_message follow social_network tags market_history account_by_key operation_dump operation_history account_history account_notes statsd block_info raw_block witness_api mongo_db # For connect to mongodb which is running outside Docker (if golosd running inside) mongodb-uri = mongodb://172.17.0.1:27017/Golos From e9c143debe02823b13568e779dad3e432a587a81 Mon Sep 17 00:00:00 2001 From: boostmultifruit Date: Tue, 7 May 2019 14:06:03 +0300 Subject: [PATCH 6/6] Fix tests for total_comment_reward_operation #1288 --- tests/tests/operation_time_tests.cpp | 41 ++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/tests/tests/operation_time_tests.cpp b/tests/tests/operation_time_tests.cpp index e003564db0..175a3ffbe2 100644 --- a/tests/tests/operation_time_tests.cpp +++ b/tests/tests/operation_time_tests.cpp @@ -364,7 +364,13 @@ BOOST_FIXTURE_TEST_SUITE(operation_time_tests, clean_database_fixture) BOOST_REQUIRE(dave_account.vesting_shares == dave_total_vesting); BOOST_REQUIRE(dave_account.sbd_balance == dave_sbd_balance); - auto bob_author_reward = get_last_operations(1)[0].get(); + auto ops = get_last_operations(2); + + auto total_reward = ops[0].get(); + BOOST_CHECK_EQUAL(total_reward.author, "bob"); + BOOST_CHECK_EQUAL(total_reward.permlink, "test"); + + auto bob_author_reward = ops[1].get(); BOOST_REQUIRE(bob_author_reward.author == "bob"); BOOST_REQUIRE(bob_author_reward.permlink == "test"); BOOST_REQUIRE(bob_author_reward.sbd_payout == bob_comment_reward.sbd_payout()); @@ -606,62 +612,75 @@ BOOST_FIXTURE_TEST_SUITE(operation_time_tests, clean_database_fixture) BOOST_CHECK(dave_cr_itr != cr_idx.end()); BOOST_CHECK_EQUAL(dave_cr_itr->total_payout_value, dave_comment_reward.total_payout()); - auto ops = get_last_operations(9); + auto ops = get_last_operations(12); BOOST_TEST_MESSAGE("Checking Virtual Operation Correctness"); curation_reward_operation vop_curation; author_reward_operation vop_author; + total_comment_reward_operation vop_total; + + vop_total = ops[0].get(); + BOOST_CHECK_EQUAL(vop_total.author, "dave"); + BOOST_CHECK_EQUAL(vop_total.permlink, "test"); - vop_author = ops[0].get(); + vop_author = ops[1].get(); BOOST_REQUIRE(vop_author.author == "dave"); BOOST_REQUIRE(vop_author.permlink == "test"); BOOST_REQUIRE(vop_author.sbd_payout == dave_comment_reward.sbd_payout()); BOOST_REQUIRE(vop_author.vesting_payout == dave_comment_reward.vesting_payout()); - vop_curation = ops[1].get(); + vop_curation = ops[2].get(); BOOST_REQUIRE(vop_curation.curator == "bob"); BOOST_REQUIRE(vop_curation.comment_author == "dave"); BOOST_REQUIRE(vop_curation.comment_permlink == "test"); BOOST_REQUIRE(vop_curation.reward == dave_comment_reward.vote_payout(bob_account)); - vop_author = ops[2].get(); + vop_total = ops[3].get(); + BOOST_CHECK_EQUAL(vop_total.author, "bob"); + BOOST_CHECK_EQUAL(vop_total.permlink, "test"); + + vop_author = ops[4].get(); BOOST_REQUIRE(vop_author.author == "bob"); BOOST_REQUIRE(vop_author.permlink == "test"); BOOST_REQUIRE(vop_author.sbd_payout == bob_comment_reward.sbd_payout()); BOOST_REQUIRE(vop_author.vesting_payout == bob_comment_reward.vesting_payout()); - vop_curation = ops[3].get(); + vop_curation = ops[5].get(); BOOST_CHECK_EQUAL(vop_curation.curator, "bob"); BOOST_CHECK_EQUAL(vop_curation.comment_author, "bob"); BOOST_CHECK_EQUAL(vop_curation.comment_permlink, "test"); BOOST_CHECK_EQUAL(vop_curation.reward, bob_comment_reward.vote_payout(bob_account)); - vop_curation = ops[4].get(); + vop_curation = ops[6].get(); BOOST_CHECK_EQUAL(vop_curation.curator, "sam"); BOOST_CHECK_EQUAL(vop_curation.comment_author, "bob"); BOOST_CHECK_EQUAL(vop_curation.comment_permlink, "test"); BOOST_CHECK_EQUAL(vop_curation.reward, bob_comment_reward.vote_payout(sam_account)); - vop_curation = ops[5].get(); + vop_curation = ops[7].get(); BOOST_CHECK_EQUAL(vop_curation.curator, "alice"); BOOST_CHECK_EQUAL(vop_curation.comment_author, "bob"); BOOST_CHECK_EQUAL(vop_curation.comment_permlink, "test"); BOOST_CHECK_EQUAL(vop_curation.reward, bob_comment_reward.vote_payout(alice_account)); - vop_author = ops[6].get(); + vop_total = ops[8].get(); + BOOST_CHECK_EQUAL(vop_total.author, "alice"); + BOOST_CHECK_EQUAL(vop_total.permlink, "test"); + + vop_author = ops[9].get(); BOOST_CHECK_EQUAL(vop_author.author, "alice"); BOOST_CHECK_EQUAL(vop_author.permlink, "test"); BOOST_CHECK_EQUAL(vop_author.sbd_payout, alice_comment_reward.sbd_payout()); BOOST_CHECK_EQUAL(vop_author.vesting_payout, alice_comment_reward.vesting_payout()); - vop_curation = ops[7].get(); + vop_curation = ops[10].get(); BOOST_CHECK_EQUAL(vop_curation.curator, "alice"); BOOST_CHECK_EQUAL(vop_curation.comment_author, "alice"); BOOST_CHECK_EQUAL(vop_curation.comment_permlink, "test"); BOOST_CHECK_EQUAL(vop_curation.reward, alice_comment_reward.vote_payout(alice_account)); - vop_curation = ops[8].get(); + vop_curation = ops[11].get(); BOOST_CHECK_EQUAL(vop_curation.curator, "bob"); BOOST_CHECK_EQUAL(vop_curation.comment_author, "alice"); BOOST_CHECK_EQUAL(vop_curation.comment_permlink, "test");