Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add bookmark functionality #330 #331

Open
wants to merge 11 commits into
base: master
Choose a base branch
from
9 changes: 8 additions & 1 deletion hive/db/db_state.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@

from hive.db.schema import (setup, reset_autovac, build_metadata,
build_metadata_community, teardown, DB_VERSION,
build_metadata_blacklist, build_trxid_block_num)
build_metadata_blacklist, build_trxid_block_num,
build_metadata_bookmarks)
from hive.db.adapter import Db

log = logging.getLogger(__name__)
Expand Down Expand Up @@ -331,6 +332,12 @@ def _check_migrations(cls):
cls.db().query("CREATE INDEX hive_block_num_ix1 ON hive_trxid_block_num (block_num)")
cls.db().query("CREATE UNIQUE INDEX hive_trxid_ix1 ON hive_trxid_block_num (trx_id) WHERE trx_id IS NOT NULL")
cls._set_ver(20)
if cls._ver == 20:
if not cls.db().query_col("SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name='hive_bookmarks')")[0]:
build_metadata_bookmarks().create_all(cls.db().engine())
cls.db().query("ALTER TABLE hive_bookmarks ADD CONSTRAINT hive_bookmarks_fk1 FOREIGN KEY (account) REFERENCES hive_accounts(name);")
cls.db().query("ALTER TABLE hive_bookmarks ADD CONSTRAINT hive_bookmarks_fk2 FOREIGN KEY (post_id) REFERENCES hive_posts(id);")
cls._set_ver(21)

reset_autovac(cls.db())

Expand Down
21 changes: 20 additions & 1 deletion hive/db/schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

#pylint: disable=line-too-long, too-many-lines, bad-whitespace

DB_VERSION = 20
DB_VERSION = 21

def build_metadata():
"""Build schema def with SqlAlchemy"""
Expand Down Expand Up @@ -243,6 +243,8 @@ def build_metadata():

metadata = build_trxid_block_num(metadata)

metadata = build_metadata_bookmarks(metadata)

return metadata

def build_metadata_community(metadata=None):
Expand Down Expand Up @@ -356,6 +358,23 @@ def build_trxid_block_num(metadata=None):

return metadata

def build_metadata_bookmarks(metadata=None):
if not metadata:
metadata = sa.MetaData()

sa.Table(
'hive_bookmarks', metadata,
sa.Column('account', VARCHAR(16), nullable=False),
sa.Column('post_id', sa.Integer, nullable=False),
sa.Column('bookmarked_at', sa.DateTime, nullable=False),

sa.UniqueConstraint('account', 'post_id', name='hive_bookmarks_ux1'),
sa.Index('hive_bookmarks_ix1', 'post_id'),
sa.Index('hive_bookmarks_ix2', 'account', 'post_id'),
sa.Index('hive_bookmarks_ix3', 'account', 'bookmarked_at'),
)

return metadata

def teardown(db):
"""Drop all tables"""
Expand Down
1 change: 1 addition & 0 deletions hive/indexer/blocks.py
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,7 @@ def _pop(cls, blocks):
DB.query("DELETE FROM hive_payments WHERE block_num = :num", num=num)
DB.query("DELETE FROM hive_blocks WHERE num = :num", num=num)
DB.query("DELETE FROM hive_trxid_block_num WHERE block_num = :num", num=num)
DB.query("DELETE FROM hive_bookmarks WHERE bookmarked_at >= :date", date=date)

DB.query("COMMIT")
log.warning("[FORK] recovery complete")
Expand Down
79 changes: 79 additions & 0 deletions hive/indexer/bookmark.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
"""Handles bookmark operations."""

import logging

from hive.db.adapter import Db
from hive.indexer.accounts import Accounts
from hive.indexer.posts import Posts

log = logging.getLogger(__name__)

DB = Db.instance()


class Bookmark:
"""Handles processing of adding and removing bookmarks and flushing to db."""

@classmethod
def bookmark_op(cls, account, op_json, date):
"""Process an incoming bookmark op."""
op = cls._validated_op(account, op_json, date)
if not op:
return

# perform add bookmark
if op['action'] == 'add':
sql = """INSERT INTO hive_bookmarks (account, post_id, bookmarked_at)
VALUES (:account, :post_id, :at)"""
DB.query(sql, **op)

# perform remove bookmark
elif op['action'] == 'remove':
sql = """DELETE FROM hive_bookmarks
WHERE account = :account AND post_id = :post_id"""
DB.query(sql, **op)

@classmethod
def _validated_op(cls, account, op, date):
"""Validate and normalize the operation."""

min_params = ['account', 'author', 'permlink', 'action', 'category']
if any(param not in op for param in min_params):
# invalid op
return None

if account != op['account']:
# impersonation
return None

if op['action'] not in ['add', 'remove']:
# invalid action
return None

account_id = Accounts.get_id(account)
if not account_id:
# invalid account
return None

post_id = Posts.get_id(op['author'], op['permlink'])
if not post_id:
# invalid post
return None

is_bookmarked = cls._is_bookmarked(account, post_id)
if ((is_bookmarked and op['action'] == 'add') # already bookmarked
or (not is_bookmarked and op['action'] == 'remove')): # not bookmarked
# invalid action
return None

return dict(account=account,
post_id=post_id,
action=op['action'],
at=date)

@classmethod
def _is_bookmarked(cls, account, post_id):
"""Return bookmark if it exists."""
sql = """SELECT 1 FROM hive_bookmarks
WHERE account = :account AND post_id = :post_id"""
return DB.query_one(sql, account=account, post_id=post_id)
12 changes: 10 additions & 2 deletions hive/indexer/custom_op.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from hive.indexer.feed_cache import FeedCache
from hive.indexer.follow import Follow
from hive.indexer.notify import Notify
from hive.indexer.bookmark import Bookmark

from hive.indexer.community import process_json_community_op, START_BLOCK
from hive.utils.normalize import load_json_key
Expand Down Expand Up @@ -78,7 +79,7 @@ def _process_notify(cls, account, op_json, block_date):

@classmethod
def _process_legacy(cls, account, op_json, block_date):
"""Handle legacy 'follow' plugin ops (follow/mute/clear, reblog)
"""Handle legacy 'follow' plugin ops (follow/mute/clear, reblog, bookmark)

follow {follower: {type: 'account'},
following: {type: 'account'},
Expand All @@ -87,12 +88,17 @@ def _process_legacy(cls, account, op_json, block_date):
author: {type: 'account'},
permlink: {type: 'permlink'},
delete: {type: 'str', optional: True}}
bookmark {account: {type: 'account'},
author: {type: 'account'},
permlink: {type: 'permlink'},
action: {type: 'str'},
category: {type: 'str'}} // category currently unused
"""
if not isinstance(op_json, list):
return
if len(op_json) != 2:
return
if first(op_json) not in ['follow', 'reblog']:
if first(op_json) not in ['follow', 'reblog', 'bookmark']:
return
if not isinstance(second(op_json), dict):
return
Expand All @@ -102,6 +108,8 @@ def _process_legacy(cls, account, op_json, block_date):
Follow.follow_op(account, op_json, block_date)
elif cmd == 'reblog':
cls.reblog(account, op_json, block_date)
elif cmd == 'bookmark':
Bookmark.bookmark_op(account, op_json, block_date)

@classmethod
def reblog(cls, account, op_json, block_date):
Expand Down
52 changes: 52 additions & 0 deletions hive/server/bridge_api/cursor.py
Original file line number Diff line number Diff line change
Expand Up @@ -475,3 +475,55 @@ async def pids_by_payout(db, account: str, start_author: str = '',
""" % seek

return await db.query_col(sql, account=account, start_id=start_id, limit=limit)

async def pids_by_bookmarks(db, account: str, sort: str = 'bookmarks', category: str = '', start_author: str = '',
start_permlink: str = '', limit: int = 20):
"""Get a list of post_ids for an author's bookmarks."""
seek = ''
join = ''
start_id = await _get_post_id(db, start_author, start_permlink) if start_permlink else None

if sort == 'bookmarks':
# order by age of bookmarks
order_by = "bookmarks.bookmarked_at DESC"
if start_permlink:
seek = """
AND bookmarks.bookmarked_at < (
SELECT bookmarked_at FROM hive_bookmarks
WHERE post_id = :start_id
AND account = :account
)
"""

elif sort == 'posts':
# order by age of posts
order_by = "bookmarks.post_id DESC"
if start_permlink:
seek = "AND bookmarks.post_id < :start_id"

elif sort == 'authors':
# order by name of authors
# sort by author, then by post_id
# for paging we need a sort if more than one post by the same author is bookmarked
join = "JOIN hive_posts AS posts ON bookmarks.post_id = posts.id"
order_by = "posts.author ASC, bookmarks.post_id DESC"
if start_permlink:
seek = """
AND (posts.author > :start_author
OR (posts.author = :start_author AND bookmarks.post_id < :start_id))
"""

sql = """
SELECT bookmarks.post_id
FROM hive_bookmarks AS bookmarks
%s
WHERE bookmarks.account = :account %s
ORDER BY %s
LIMIT :limit
""" % (join, seek, order_by)

return await db.query_col(sql,
account=account,
start_id=start_id,
start_author=start_author,
limit=limit)
33 changes: 33 additions & 0 deletions hive/server/bridge_api/methods.py
Original file line number Diff line number Diff line change
Expand Up @@ -156,3 +156,36 @@ async def get_account_posts(context, sort, account, start_author='', start_perml
if pid in ids:
ids.remove(pid)
return await load_posts(context['db'], ids)

@return_error_info
async def get_bookmarked_posts(context, account, sort='bookmarks', category='', start_author='', start_permlink='',
limit=20, observer=None):
"""Get bookmarked posts for an account"""

# valid sorts are by age of posts, age of bookmarks or authors
valid_sorts = ['posts', 'bookmarks', 'authors']
assert sort in valid_sorts, 'invalid bookmark sort'
assert account, 'account is required'

db = context['db']
account = valid_account(account)
start_author = valid_account(start_author, allow_empty=True)
start_permlink = valid_permlink(start_permlink, allow_empty=True)
start = (start_author, start_permlink)
limit = valid_limit(limit, 50)
category = '' # currently unused

# check blacklist accounts
_id = await db.query_one("SELECT id FROM hive_posts_status WHERE author = :n AND list_type = '3'", n=account)
if _id:
return []

ids = await cursor.pids_by_bookmarks(
context['db'],
account,
sort,
category,
*start,
limit)

return await load_posts(context['db'], ids)
10 changes: 10 additions & 0 deletions hive/server/bridge_api/objects.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,14 @@ async def load_posts_keyed(db, ids, truncate_body=0):
ctx[cid] = []
ctx[cid].append(author['id'])

# fetch bookmarks
sql = """SELECT post_id, string_agg(account, ',') AS bookmarked_by
FROM hive_bookmarks
WHERE post_id IN :ids
GROUP BY post_id"""
bookmarks = await db.query_all(sql, ids=tuple(ids))
bookmarks = {row[0]:row[1].split(',') for row in bookmarks}

# TODO: optimize
titles = {}
roles = {}
Expand Down Expand Up @@ -106,6 +114,8 @@ async def load_posts_keyed(db, ids, truncate_body=0):
or len(post['blacklists']) >= 2)
post['stats']['hide'] = 'irredeemables' in post['blacklists']

# add bookmarked account names
post["bookmarked_by"] = bookmarks.get(pid, [])

sql = """SELECT id FROM hive_posts
WHERE id IN :ids AND is_pinned = '1' AND is_deleted = '0'"""
Expand Down
1 change: 1 addition & 0 deletions hive/server/serve.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,7 @@ def build_methods():
bridge_api.get_ranked_posts,
bridge_api.get_profile,
bridge_api.get_trending_topics,
bridge_api.get_bookmarked_posts,
hive_api_notify.post_notifications,
hive_api_notify.account_notifications,
hive_api_notify.unread_notifications,
Expand Down