From 68c36eb845fc5b5d0a28f9e3cfda5664ea068211 Mon Sep 17 00:00:00 2001 From: roadscape Date: Wed, 13 Mar 2019 15:50:15 -0500 Subject: [PATCH 1/3] add mutes functionality --- hive/conf.py | 1 + hive/server/common/mutes.py | 33 ++++++++++++++++++++++++++ hive/server/condenser_api/get_state.py | 7 +++++- hive/server/condenser_api/objects.py | 8 +++++++ hive/server/serve.py | 4 ++++ 5 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 hive/server/common/mutes.py diff --git a/hive/conf.py b/hive/conf.py index 916f56cd2..bad6b6d60 100644 --- a/hive/conf.py +++ b/hive/conf.py @@ -33,6 +33,7 @@ def init_argparse(cls, strict=True, **kwargs): # common add('--database-url', env_var='DATABASE_URL', required=False, help='database connection url', default='') add('--steemd-url', env_var='STEEMD_URL', required=False, help='steemd/jussi endpoint', default='https://api.steemit.com') + add('--muted-accounts-url', env_var='MUTED_ACCOUNTS_URL', required=False, help='url to flat list of muted accounts', default='') # server add('--http-server-port', type=int, env_var='HTTP_SERVER_PORT', default=8080) diff --git a/hive/server/common/mutes.py b/hive/server/common/mutes.py new file mode 100644 index 000000000..2e5d8124f --- /dev/null +++ b/hive/server/common/mutes.py @@ -0,0 +1,33 @@ +"""List of muted accounts for server process.""" + +import logging +from urllib.request import urlopen + +log = logging.getLogger(__name__) + +class Mutes: + """Singleton tracking muted accounts.""" + + _instance = None + accounts = set() + + @classmethod + def instance(cls): + """Get the shared instance.""" + assert cls._instance, 'set_shared_instance was never called' + return cls._instance + + @classmethod + def set_shared_instance(cls, instance): + """Set the global/shared instance.""" + cls._instance = instance + + def __init__(self, url): + """Initialize a muted account list by loading from URL""" + if url: + self.accounts = set(urlopen(url).read().decode('utf8').split()) + + @classmethod + def all(cls): + """Return the set of all muted accounts from singleton instance.""" + return cls.instance().accounts diff --git a/hive/server/condenser_api/get_state.py b/hive/server/condenser_api/get_state.py index 5d757eb4f..b7419b1dd 100644 --- a/hive/server/condenser_api/get_state.py +++ b/hive/server/condenser_api/get_state.py @@ -6,6 +6,7 @@ import ujson as json from hive.utils.normalize import legacy_amount +from hive.server.common.mutes import Mutes from hive.server.condenser_api.objects import ( load_accounts, @@ -255,9 +256,13 @@ async def _load_discussion(db, author, permlink): posts = await load_posts_keyed(db, ids) refs = {pid: _ref(post) for pid, post in posts.items()} + muted_accounts = Mutes.all() + # add child refs to parent posts for pid, post in posts.items(): - if pid in tree: + if post['author'] in muted_accounts: + del posts[pid] + elif pid in tree: post['replies'] = [refs[cid] for cid in tree[pid] if cid in refs] diff --git a/hive/server/condenser_api/objects.py b/hive/server/condenser_api/objects.py index edc63ec39..d8ee9a7ea 100644 --- a/hive/server/condenser_api/objects.py +++ b/hive/server/condenser_api/objects.py @@ -4,6 +4,7 @@ import ujson as json from hive.utils.normalize import sbd_amount, rep_to_raw +from hive.server.common.mutes import Mutes log = logging.getLogger(__name__) @@ -45,15 +46,22 @@ async def load_posts_keyed(db, ids, truncate_body=0): result = await db.query_all(sql, ids=tuple(ids)) author_reps = await _query_author_rep_map(db, result) + muted_accounts = Mutes.all() posts_by_id = {} for row in result: row = dict(row) row['author_rep'] = author_reps[row['author']] post = _condenser_post_object(row, truncate_body=truncate_body) + post['active_votes'] = _mute_votes(post['active_votes'], muted_accounts) posts_by_id[row['post_id']] = post return posts_by_id +def _mute_votes(votes, muted_accounts): + if not muted_accounts: + return votes + return [v for v in votes if v['voter'] not in muted_accounts] + async def load_posts(db, ids, truncate_body=0): """Given an array of post ids, returns full objects in the same order.""" if not ids: diff --git a/hive/server/serve.py b/hive/server/serve.py index 923c132be..6e3636146 100644 --- a/hive/server/serve.py +++ b/hive/server/serve.py @@ -15,6 +15,7 @@ from hive.server.condenser_api.get_state import get_state as condenser_api_get_state from hive.server.condenser_api.call import call as condenser_api_call from hive.server import hive_api +from hive.server.common.mutes import Mutes from hive.server.db import Db @@ -124,6 +125,9 @@ def run_server(conf): log = logging.getLogger(__name__) methods = build_methods() + mutes = Mutes(conf.get('muted_accounts_url')) + Mutes.set_shared_instance(mutes) + app = web.Application() app['config'] = dict() app['config']['args'] = conf.args() From 82c4ce4bd70f04c3a7b50aa43b93e46810f31bc8 Mon Sep 17 00:00:00 2001 From: roadscape Date: Wed, 13 Mar 2019 16:18:02 -0500 Subject: [PATCH 2/3] del items recursively --- hive/server/condenser_api/get_state.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/hive/server/condenser_api/get_state.py b/hive/server/condenser_api/get_state.py index b7419b1dd..0ac878287 100644 --- a/hive/server/condenser_api/get_state.py +++ b/hive/server/condenser_api/get_state.py @@ -257,15 +257,23 @@ async def _load_discussion(db, author, permlink): refs = {pid: _ref(post) for pid, post in posts.items()} muted_accounts = Mutes.all() + rem_pids = [] # add child refs to parent posts for pid, post in posts.items(): if post['author'] in muted_accounts: - del posts[pid] + rem_pids.append(pid) elif pid in tree: post['replies'] = [refs[cid] for cid in tree[pid] if cid in refs] + # remove posts/comments from muted accounts + for pid in rem_pids: + if pid in posts: + del posts[pid] + if pid in tree: + rem_pids.extend(tree[pid]) + # return all nodes keyed by ref return {refs[pid]: post for pid, post in posts.items()} From 0cf8313a17e381aa277d56265c5b1af0a9df4453 Mon Sep 17 00:00:00 2001 From: roadscape Date: Wed, 13 Mar 2019 16:56:32 -0500 Subject: [PATCH 3/3] prune posts eagerly --- hive/server/condenser_api/get_state.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/hive/server/condenser_api/get_state.py b/hive/server/condenser_api/get_state.py index 0ac878287..26a92845f 100644 --- a/hive/server/condenser_api/get_state.py +++ b/hive/server/condenser_api/get_state.py @@ -254,26 +254,27 @@ async def _load_discussion(db, author, permlink): # load all post objects, build ref-map posts = await load_posts_keyed(db, ids) - refs = {pid: _ref(post) for pid, post in posts.items()} + # remove posts/comments from muted accounts muted_accounts = Mutes.all() rem_pids = [] - - # add child refs to parent posts for pid, post in posts.items(): if post['author'] in muted_accounts: rem_pids.append(pid) - elif pid in tree: - post['replies'] = [refs[cid] for cid in tree[pid] - if cid in refs] - - # remove posts/comments from muted accounts for pid in rem_pids: if pid in posts: del posts[pid] if pid in tree: rem_pids.extend(tree[pid]) + refs = {pid: _ref(post) for pid, post in posts.items()} + + # add child refs to parent posts + for pid, post in posts.items(): + if pid in tree: + post['replies'] = [refs[cid] for cid in tree[pid] + if cid in refs] + # return all nodes keyed by ref return {refs[pid]: post for pid, post in posts.items()}