From 4700bb1b4c3fd8ba8c8decf25c75bb50120c1c92 Mon Sep 17 00:00:00 2001 From: only-dev-time <111915936+only-dev-time@users.noreply.github.com> Date: Fri, 19 Jan 2024 16:42:29 +0100 Subject: [PATCH 01/10] add table hive_bookmarks to database schema (metadata indizes foreign keys) --- hive/db/db_state.py | 9 ++++++++- hive/db/schema.py | 21 ++++++++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/hive/db/db_state.py b/hive/db/db_state.py index 519fb8df..f0615651 100644 --- a/hive/db/db_state.py +++ b/hive/db/db_state.py @@ -7,7 +7,8 @@ from hive.db.schema import (setup, reset_autovac, build_metadata, build_metadata_community, teardown, DB_VERSION, - build_metadata_blacklist, build_trxid_block_num) + build_metadata_blacklist, build_trxid_block_num, + build_metadata_bookmarks) from hive.db.adapter import Db log = logging.getLogger(__name__) @@ -331,6 +332,12 @@ def _check_migrations(cls): cls.db().query("CREATE INDEX hive_block_num_ix1 ON hive_trxid_block_num (block_num)") cls.db().query("CREATE UNIQUE INDEX hive_trxid_ix1 ON hive_trxid_block_num (trx_id) WHERE trx_id IS NOT NULL") cls._set_ver(20) + if cls._ver == 20: + if not cls.db().query_col("SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name='hive_bookmarks')")[0]: + build_metadata_bookmarks().create_all(cls.db().engine()) + cls.db().query("ALTER TABLE hive_bookmarks ADD CONSTRAINT hive_bookmarks_fk1 FOREIGN KEY (account_id) REFERENCES hive_accounts(id);") + cls.db().query("ALTER TABLE hive_bookmarks ADD CONSTRAINT hive_bookmarks_fk2 FOREIGN KEY (post_id) REFERENCES hive_posts(id);") + cls._set_ver(21) reset_autovac(cls.db()) diff --git a/hive/db/schema.py b/hive/db/schema.py index ef406d86..06d762e8 100644 --- a/hive/db/schema.py +++ b/hive/db/schema.py @@ -10,7 +10,7 @@ #pylint: disable=line-too-long, too-many-lines, bad-whitespace -DB_VERSION = 20 +DB_VERSION = 21 def build_metadata(): """Build schema def with SqlAlchemy""" @@ -243,6 +243,8 @@ def build_metadata(): metadata = build_trxid_block_num(metadata) + metadata = build_metadata_bookmarks(metadata) + return metadata def build_metadata_community(metadata=None): @@ -356,6 +358,23 @@ def build_trxid_block_num(metadata=None): return metadata +def build_metadata_bookmarks(metadata=None): + if not metadata: + metadata = sa.MetaData() + + sa.Table( + 'hive_bookmarks', metadata, + sa.Column('id', sa.Integer, primary_key=True, autoincrement=True), # TODO notwendig? + sa.Column('account_id', sa.Integer, nullable=False), + sa.Column('post_id', sa.Integer, nullable=False), + sa.Column('bookmarked_at', sa.DateTime, nullable=False), + + sa.UniqueConstraint('account_id', 'post_id', name='hive_bookmarks_ux1'), + sa.Index('hive_bookmarks_ix1', 'account_id'), # bookmarks from account + sa.Index('hive_bookmarks_ix2', 'post_id'), # bookmarks for a post + ) + + return metadata def teardown(db): """Drop all tables""" From 1896be8a1bbdfcd317593b9cc54f424a096e54b5 Mon Sep 17 00:00:00 2001 From: only-dev-time <111915936+only-dev-time@users.noreply.github.com> Date: Fri, 19 Jan 2024 22:36:00 +0100 Subject: [PATCH 02/10] extend indexer for bookmark functionality --- hive/indexer/bookmark.py | 68 +++++++++++++++++++++++++++++++++++++++ hive/indexer/custom_op.py | 12 +++++-- 2 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 hive/indexer/bookmark.py diff --git a/hive/indexer/bookmark.py b/hive/indexer/bookmark.py new file mode 100644 index 00000000..5a68b279 --- /dev/null +++ b/hive/indexer/bookmark.py @@ -0,0 +1,68 @@ +"""Handles bookmark operations.""" + +import logging + +from hive.db.adapter import Db +from hive.indexer.accounts import Accounts +from hive.indexer.posts import Posts + +log = logging.getLogger(__name__) + +DB = Db.instance() + + +class Bookmark: + """Handles processing of adding and removing bookmarks and flushing to db.""" + + @classmethod + def bookmark_op(cls, account, op_json, date): + """Process an incoming bookmark op.""" + op = cls._validated_op(account, op_json, date) + if not op: + return + + # perform add bookmark + if op['action'] == 'add': + sql = """INSERT INTO hive_bookmarks (account_id, post_id, created_at) + VALUES (:account_id, :post_id, :at)""" + DB.query(sql, **op) + # TODO notify author of bookmark added + # TODO update bookmarks count on post + + # perform remove bookmark + elif op['action'] == 'remove': + sql = """DELETE FROM hive_bookmarks + WHERE account_id = :account_id AND post_id = :post_id""" + DB.query(sql, **op) + # TODO update bookmarks count on post + + @classmethod + def _validated_op(cls, account, op, date): + """Validate and normalize the operation.""" + + min_params = ['account', 'author', 'permlink', 'action', 'category'] + if any(param not in op for param in min_params): + # invalid op + return None + + if account != op['account']: + # impersonation + return None + + if op['action'] not in ['add', 'remove']: + # invalid action + return None + + if not Accounts.exists(account): + # invalid account + return None + + post_id = Posts.get_id(op['author'], op['permlink']) + if not post_id: + # invalid post + return None + + return dict(account_id=Accounts.get_id(account), + post_id=post_id, + action=op['action'], + at=date) diff --git a/hive/indexer/custom_op.py b/hive/indexer/custom_op.py index 995813e8..564d6949 100644 --- a/hive/indexer/custom_op.py +++ b/hive/indexer/custom_op.py @@ -10,6 +10,7 @@ from hive.indexer.feed_cache import FeedCache from hive.indexer.follow import Follow from hive.indexer.notify import Notify +from hive.indexer.bookmark import Bookmark from hive.indexer.community import process_json_community_op, START_BLOCK from hive.utils.normalize import load_json_key @@ -78,7 +79,7 @@ def _process_notify(cls, account, op_json, block_date): @classmethod def _process_legacy(cls, account, op_json, block_date): - """Handle legacy 'follow' plugin ops (follow/mute/clear, reblog) + """Handle legacy 'follow' plugin ops (follow/mute/clear, reblog, bookmark) follow {follower: {type: 'account'}, following: {type: 'account'}, @@ -87,12 +88,17 @@ def _process_legacy(cls, account, op_json, block_date): author: {type: 'account'}, permlink: {type: 'permlink'}, delete: {type: 'str', optional: True}} + bookmark {account: {type: 'account'}, + author: {type: 'account'}, + permlink: {type: 'permlink'}, + action: {type: 'str'}, + category: {type: 'str'}} """ if not isinstance(op_json, list): return if len(op_json) != 2: return - if first(op_json) not in ['follow', 'reblog']: + if first(op_json) not in ['follow', 'reblog', 'bookmark']: return if not isinstance(second(op_json), dict): return @@ -102,6 +108,8 @@ def _process_legacy(cls, account, op_json, block_date): Follow.follow_op(account, op_json, block_date) elif cmd == 'reblog': cls.reblog(account, op_json, block_date) + elif cmd == 'bookmark': + Bookmark.bookmark_op(account, op_json, block_date) @classmethod def reblog(cls, account, op_json, block_date): From 1ac8b55f0e0c59a85f062aab4d5a3d1794084f5d Mon Sep 17 00:00:00 2001 From: only-dev-time <111915936+only-dev-time@users.noreply.github.com> Date: Sat, 20 Jan 2024 22:44:19 +0100 Subject: [PATCH 03/10] add get_bookmarked_posts to bridge_api --- hive/server/bridge_api/cursor.py | 37 +++++++++++++++++++++++++++++++ hive/server/bridge_api/methods.py | 32 ++++++++++++++++++++++++++ hive/server/serve.py | 1 + 3 files changed, 70 insertions(+) diff --git a/hive/server/bridge_api/cursor.py b/hive/server/bridge_api/cursor.py index d29f4a4e..088ae901 100644 --- a/hive/server/bridge_api/cursor.py +++ b/hive/server/bridge_api/cursor.py @@ -465,3 +465,40 @@ async def pids_by_payout(db, account: str, start_author: str = '', """ % seek return await db.query_col(sql, account=account, start_id=start_id, limit=limit) + +async def pids_by_bookmarks(db, account: str, sort: str = 'bookmark', category: str = '', start_author: str = '', + start_permlink: str = '', limit: int = 20): + """Get a list of post_ids for an author's bookmarks.""" + account_id = await _get_account_id(db, account) + seek = '' + join = '' + start_id = None + + if sort == 'bookmark': + # order by age of bookmarks + order_by = "bookmarks.bookmarked_at DESC" + # if start_permlink: + # start_id = # TODO start_id ermitteln aus bookmarks-table - ggf. get_bookmark_id() verwenden und neuen Index in Tabelle anlegen... + elif sort == 'post': + # order by age of posts + order_by = "bookmarks.post_id DESC" + # if start_permlink: + # start_id = await _get_post_id(db, start_author, start_permlink) + # seek = "AND post_id <= :start_id" + elif sort == 'author': + # order by name of authors + join = "JOIN hive_posts AS posts ON bookmarks.post_id = posts.id" + order_by = "posts.author ASC" + # if start_permlink: + # seek = "AND authors.name >= :start_author AND posts.permlink >= :start_permlink" # TODO testen + + sql = """ + SELECT bookmarks.post_id + FROM hive_bookmarks AS bookmarks + %s + WHERE account_id = :account_id %s + ORDER BY %s + LIMIT :limit + """ % (join, seek, order_by) + + return await db.query_col(sql, account_id=account_id, start_id=start_id, limit=limit) \ No newline at end of file diff --git a/hive/server/bridge_api/methods.py b/hive/server/bridge_api/methods.py index 4d4d9c44..4386031c 100644 --- a/hive/server/bridge_api/methods.py +++ b/hive/server/bridge_api/methods.py @@ -156,3 +156,35 @@ async def get_account_posts(context, sort, account, start_author='', start_perml if pid in ids: ids.remove(pid) return await load_posts(context['db'], ids) + +@return_error_info +async def get_bookmarked_posts(context, account, sort='bookmark', category='', start_author='', start_permlink='', + limit=20, observer=None): + """Get bookmarked posts for an account""" + + # valid sorts are by age of posts, age of bookmarks or authors + valid_sorts = ['post', 'bookmark', 'author'] + assert sort in valid_sorts, 'invalid bookmark sort' + assert account, 'account is required' + + db = context['db'] + account = valid_account(account) + start_author = valid_account(start_author, allow_empty=True) + start_permlink = valid_permlink(start_permlink, allow_empty=True) + start = (start_author, start_permlink) + limit = valid_limit(limit, 100) + + # check blacklist accounts + _id = await db.query_one("SELECT id FROM hive_posts_status WHERE author = :n AND list_type = '3'", n=account) + if _id: + return [] + + ids = await cursor.pids_by_bookmarks( + context['db'], + account, + sort, + category, + *start, + limit) + + return await load_posts(context['db'], ids) \ No newline at end of file diff --git a/hive/server/serve.py b/hive/server/serve.py index 09740664..d59d3c84 100644 --- a/hive/server/serve.py +++ b/hive/server/serve.py @@ -127,6 +127,7 @@ def build_methods(): bridge_api.get_ranked_posts, bridge_api.get_profile, bridge_api.get_trending_topics, + bridge_api.get_bookmarked_posts, hive_api_notify.post_notifications, hive_api_notify.account_notifications, hive_api_notify.unread_notifications, From ace8eecf5da4de0797003fd368a92b72bff08b5f Mon Sep 17 00:00:00 2001 From: only-dev-time <111915936+only-dev-time@users.noreply.github.com> Date: Sat, 20 Jan 2024 22:44:55 +0100 Subject: [PATCH 04/10] handle forks and fix indexer --- hive/indexer/blocks.py | 1 + hive/indexer/bookmark.py | 20 +++++++++++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/hive/indexer/blocks.py b/hive/indexer/blocks.py index 484b0526..90406007 100644 --- a/hive/indexer/blocks.py +++ b/hive/indexer/blocks.py @@ -229,6 +229,7 @@ def _pop(cls, blocks): DB.query("DELETE FROM hive_payments WHERE block_num = :num", num=num) DB.query("DELETE FROM hive_blocks WHERE num = :num", num=num) DB.query("DELETE FROM hive_trxid_block_num WHERE block_num = :num", num=num) + DB.query("DELETE FROM hive_bookmarks WHERE bookmarked_at >= :date", date=date) DB.query("COMMIT") log.warning("[FORK] recovery complete") diff --git a/hive/indexer/bookmark.py b/hive/indexer/bookmark.py index 5a68b279..934af857 100644 --- a/hive/indexer/bookmark.py +++ b/hive/indexer/bookmark.py @@ -23,7 +23,7 @@ def bookmark_op(cls, account, op_json, date): # perform add bookmark if op['action'] == 'add': - sql = """INSERT INTO hive_bookmarks (account_id, post_id, created_at) + sql = """INSERT INTO hive_bookmarks (account_id, post_id, bookmarked_at) VALUES (:account_id, :post_id, :at)""" DB.query(sql, **op) # TODO notify author of bookmark added @@ -53,7 +53,8 @@ def _validated_op(cls, account, op, date): # invalid action return None - if not Accounts.exists(account): + account_id = Accounts.get_id(account) + if not account_id: # invalid account return None @@ -62,7 +63,20 @@ def _validated_op(cls, account, op, date): # invalid post return None - return dict(account_id=Accounts.get_id(account), + exist_bookmark = cls._get_bookmark(account_id, post_id) + if ((exist_bookmark and op['action'] == 'add') # already bookmarked + or (not exist_bookmark and op['action'] == 'remove')): # not bookmarked + # invalid action + return None + + return dict(account_id=account_id, post_id=post_id, action=op['action'], at=date) + + @classmethod + def _get_bookmark(cls, account_id, post_id): + """Return bookmark if it exists.""" + sql = """SELECT 1 FROM hive_bookmarks + WHERE account_id = :account_id AND post_id = :post_id""" + return DB.query_one(sql, account_id=account_id, post_id=post_id) \ No newline at end of file From 19dbb5a53949698da8589184e3e5c3d83417a6ca Mon Sep 17 00:00:00 2001 From: only-dev-time <111915936+only-dev-time@users.noreply.github.com> Date: Fri, 2 Feb 2024 15:02:04 +0100 Subject: [PATCH 05/10] fix sort parameter options for get_bookmarked_posts --- hive/server/bridge_api/cursor.py | 8 ++++---- hive/server/bridge_api/methods.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/hive/server/bridge_api/cursor.py b/hive/server/bridge_api/cursor.py index 088ae901..7a560d97 100644 --- a/hive/server/bridge_api/cursor.py +++ b/hive/server/bridge_api/cursor.py @@ -466,7 +466,7 @@ async def pids_by_payout(db, account: str, start_author: str = '', return await db.query_col(sql, account=account, start_id=start_id, limit=limit) -async def pids_by_bookmarks(db, account: str, sort: str = 'bookmark', category: str = '', start_author: str = '', +async def pids_by_bookmarks(db, account: str, sort: str = 'bookmarks', category: str = '', start_author: str = '', start_permlink: str = '', limit: int = 20): """Get a list of post_ids for an author's bookmarks.""" account_id = await _get_account_id(db, account) @@ -474,18 +474,18 @@ async def pids_by_bookmarks(db, account: str, sort: str = 'bookmark', category: join = '' start_id = None - if sort == 'bookmark': + if sort == 'bookmarks': # order by age of bookmarks order_by = "bookmarks.bookmarked_at DESC" # if start_permlink: # start_id = # TODO start_id ermitteln aus bookmarks-table - ggf. get_bookmark_id() verwenden und neuen Index in Tabelle anlegen... - elif sort == 'post': + elif sort == 'posts': # order by age of posts order_by = "bookmarks.post_id DESC" # if start_permlink: # start_id = await _get_post_id(db, start_author, start_permlink) # seek = "AND post_id <= :start_id" - elif sort == 'author': + elif sort == 'authors': # order by name of authors join = "JOIN hive_posts AS posts ON bookmarks.post_id = posts.id" order_by = "posts.author ASC" diff --git a/hive/server/bridge_api/methods.py b/hive/server/bridge_api/methods.py index 4386031c..ae5f2a5b 100644 --- a/hive/server/bridge_api/methods.py +++ b/hive/server/bridge_api/methods.py @@ -158,12 +158,12 @@ async def get_account_posts(context, sort, account, start_author='', start_perml return await load_posts(context['db'], ids) @return_error_info -async def get_bookmarked_posts(context, account, sort='bookmark', category='', start_author='', start_permlink='', +async def get_bookmarked_posts(context, account, sort='bookmarks', category='', start_author='', start_permlink='', limit=20, observer=None): """Get bookmarked posts for an account""" # valid sorts are by age of posts, age of bookmarks or authors - valid_sorts = ['post', 'bookmark', 'author'] + valid_sorts = ['posts', 'bookmarks', 'authors'] assert sort in valid_sorts, 'invalid bookmark sort' assert account, 'account is required' From 72dc2b40d67bd060aed84766415c2c40c783d6a6 Mon Sep 17 00:00:00 2001 From: only-dev-time <111915936+only-dev-time@users.noreply.github.com> Date: Fri, 2 Feb 2024 23:46:13 +0100 Subject: [PATCH 06/10] add sorting and pagination for pids_by_bookmarks --- hive/server/bridge_api/cursor.py | 35 +++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/hive/server/bridge_api/cursor.py b/hive/server/bridge_api/cursor.py index 7a560d97..44e59d38 100644 --- a/hive/server/bridge_api/cursor.py +++ b/hive/server/bridge_api/cursor.py @@ -472,25 +472,36 @@ async def pids_by_bookmarks(db, account: str, sort: str = 'bookmarks', category: account_id = await _get_account_id(db, account) seek = '' join = '' - start_id = None + start_id = await _get_post_id(db, start_author, start_permlink) if start_permlink else None if sort == 'bookmarks': # order by age of bookmarks order_by = "bookmarks.bookmarked_at DESC" - # if start_permlink: - # start_id = # TODO start_id ermitteln aus bookmarks-table - ggf. get_bookmark_id() verwenden und neuen Index in Tabelle anlegen... + if start_permlink: + join = "JOIN hive_posts AS posts ON bookmarks.post_id = posts.id" + seek = """ + AND bookmarks.bookmarked_at < ( + SELECT bookmarked_at FROM hive_bookmarks WHERE post_id = :start_id + ) + """ + elif sort == 'posts': # order by age of posts order_by = "bookmarks.post_id DESC" - # if start_permlink: - # start_id = await _get_post_id(db, start_author, start_permlink) - # seek = "AND post_id <= :start_id" + if start_permlink: + seek = "AND bookmarks.post_id < :start_id" + elif sort == 'authors': # order by name of authors + # sort by author, then by post_id + # for paging we need a sort if more than one post by the same author is bookmarked join = "JOIN hive_posts AS posts ON bookmarks.post_id = posts.id" - order_by = "posts.author ASC" - # if start_permlink: - # seek = "AND authors.name >= :start_author AND posts.permlink >= :start_permlink" # TODO testen + order_by = "posts.author ASC, bookmarks.post_id DESC" + if start_permlink: + seek = """ + AND (posts.author > :start_author + OR (posts.author = :start_author AND bookmarks.post_id < :start_id)) + """ sql = """ SELECT bookmarks.post_id @@ -501,4 +512,8 @@ async def pids_by_bookmarks(db, account: str, sort: str = 'bookmarks', category: LIMIT :limit """ % (join, seek, order_by) - return await db.query_col(sql, account_id=account_id, start_id=start_id, limit=limit) \ No newline at end of file + return await db.query_col(sql, + account_id=account_id, + start_id=start_id, + start_author=start_author, + limit=limit) \ No newline at end of file From eba6539cd59f4c26d518eb67a92c64b74ac58eef Mon Sep 17 00:00:00 2001 From: only-dev-time <111915936+only-dev-time@users.noreply.github.com> Date: Mon, 26 Feb 2024 14:39:48 +0100 Subject: [PATCH 07/10] add field bookmarked_by to loaded post data --- hive/server/bridge_api/objects.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/hive/server/bridge_api/objects.py b/hive/server/bridge_api/objects.py index 13ee2b11..664c6705 100644 --- a/hive/server/bridge_api/objects.py +++ b/hive/server/bridge_api/objects.py @@ -75,6 +75,14 @@ async def load_posts_keyed(db, ids, truncate_body=0): ctx[cid] = [] ctx[cid].append(author['id']) + # fetch bookmarks + sql = """SELECT post_id, string_agg(account, ',') AS bookmarked_by + FROM hive_bookmarks + WHERE post_id IN :ids + GROUP BY post_id""" + bookmarks = await db.query_all(sql, ids=tuple(ids)) + bookmarks = {row[0]:row[1].split(',') for row in bookmarks} + # TODO: optimize titles = {} roles = {} @@ -105,6 +113,8 @@ async def load_posts_keyed(db, ids, truncate_body=0): or len(post['blacklists']) >= 2) post['stats']['hide'] = 'irredeemables' in post['blacklists'] + # add bookmarked account names + post["bookmarked_by"] = bookmarks.get(pid, []) sql = """SELECT id FROM hive_posts WHERE id IN :ids AND is_pinned = '1' AND is_deleted = '0'""" From ca786f4afc837b690ada2de7f9a3c429511052ac Mon Sep 17 00:00:00 2001 From: only-dev-time <111915936+only-dev-time@users.noreply.github.com> Date: Mon, 26 Feb 2024 16:23:49 +0100 Subject: [PATCH 08/10] use account name instead of id in table hive_bookmarks --- hive/db/db_state.py | 2 +- hive/db/schema.py | 6 +++--- hive/indexer/bookmark.py | 20 ++++++++++---------- hive/server/bridge_api/cursor.py | 4 ++-- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/hive/db/db_state.py b/hive/db/db_state.py index f0615651..72a0d26a 100644 --- a/hive/db/db_state.py +++ b/hive/db/db_state.py @@ -335,7 +335,7 @@ def _check_migrations(cls): if cls._ver == 20: if not cls.db().query_col("SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name='hive_bookmarks')")[0]: build_metadata_bookmarks().create_all(cls.db().engine()) - cls.db().query("ALTER TABLE hive_bookmarks ADD CONSTRAINT hive_bookmarks_fk1 FOREIGN KEY (account_id) REFERENCES hive_accounts(id);") + cls.db().query("ALTER TABLE hive_bookmarks ADD CONSTRAINT hive_bookmarks_fk1 FOREIGN KEY (account) REFERENCES hive_accounts(name);") cls.db().query("ALTER TABLE hive_bookmarks ADD CONSTRAINT hive_bookmarks_fk2 FOREIGN KEY (post_id) REFERENCES hive_posts(id);") cls._set_ver(21) diff --git a/hive/db/schema.py b/hive/db/schema.py index 06d762e8..5dab9cec 100644 --- a/hive/db/schema.py +++ b/hive/db/schema.py @@ -365,12 +365,12 @@ def build_metadata_bookmarks(metadata=None): sa.Table( 'hive_bookmarks', metadata, sa.Column('id', sa.Integer, primary_key=True, autoincrement=True), # TODO notwendig? - sa.Column('account_id', sa.Integer, nullable=False), + sa.Column('account', VARCHAR(16), nullable=False), sa.Column('post_id', sa.Integer, nullable=False), sa.Column('bookmarked_at', sa.DateTime, nullable=False), - sa.UniqueConstraint('account_id', 'post_id', name='hive_bookmarks_ux1'), - sa.Index('hive_bookmarks_ix1', 'account_id'), # bookmarks from account + sa.UniqueConstraint('account', 'post_id', name='hive_bookmarks_ux1'), + sa.Index('hive_bookmarks_ix1', 'account'), # bookmarks from account sa.Index('hive_bookmarks_ix2', 'post_id'), # bookmarks for a post ) diff --git a/hive/indexer/bookmark.py b/hive/indexer/bookmark.py index 934af857..1e16f0f7 100644 --- a/hive/indexer/bookmark.py +++ b/hive/indexer/bookmark.py @@ -23,8 +23,8 @@ def bookmark_op(cls, account, op_json, date): # perform add bookmark if op['action'] == 'add': - sql = """INSERT INTO hive_bookmarks (account_id, post_id, bookmarked_at) - VALUES (:account_id, :post_id, :at)""" + sql = """INSERT INTO hive_bookmarks (account, post_id, bookmarked_at) + VALUES (:account, :post_id, :at)""" DB.query(sql, **op) # TODO notify author of bookmark added # TODO update bookmarks count on post @@ -32,7 +32,7 @@ def bookmark_op(cls, account, op_json, date): # perform remove bookmark elif op['action'] == 'remove': sql = """DELETE FROM hive_bookmarks - WHERE account_id = :account_id AND post_id = :post_id""" + WHERE account = :account AND post_id = :post_id""" DB.query(sql, **op) # TODO update bookmarks count on post @@ -63,20 +63,20 @@ def _validated_op(cls, account, op, date): # invalid post return None - exist_bookmark = cls._get_bookmark(account_id, post_id) - if ((exist_bookmark and op['action'] == 'add') # already bookmarked - or (not exist_bookmark and op['action'] == 'remove')): # not bookmarked + is_bookmarked = cls._is_bookmarked(account, post_id) + if ((is_bookmarked and op['action'] == 'add') # already bookmarked + or (not is_bookmarked and op['action'] == 'remove')): # not bookmarked # invalid action return None - return dict(account_id=account_id, + return dict(account=account, post_id=post_id, action=op['action'], at=date) @classmethod - def _get_bookmark(cls, account_id, post_id): + def _is_bookmarked(cls, account, post_id): """Return bookmark if it exists.""" sql = """SELECT 1 FROM hive_bookmarks - WHERE account_id = :account_id AND post_id = :post_id""" - return DB.query_one(sql, account_id=account_id, post_id=post_id) \ No newline at end of file + WHERE account = :account AND post_id = :post_id""" + return DB.query_one(sql, account=account, post_id=post_id) \ No newline at end of file diff --git a/hive/server/bridge_api/cursor.py b/hive/server/bridge_api/cursor.py index 44e59d38..91d14d07 100644 --- a/hive/server/bridge_api/cursor.py +++ b/hive/server/bridge_api/cursor.py @@ -507,13 +507,13 @@ async def pids_by_bookmarks(db, account: str, sort: str = 'bookmarks', category: SELECT bookmarks.post_id FROM hive_bookmarks AS bookmarks %s - WHERE account_id = :account_id %s + WHERE account = :account %s ORDER BY %s LIMIT :limit """ % (join, seek, order_by) return await db.query_col(sql, - account_id=account_id, + account=account, start_id=start_id, start_author=start_author, limit=limit) \ No newline at end of file From c8633037390f2f832159d49f89a17d0f31f5c8d9 Mon Sep 17 00:00:00 2001 From: only-dev-time <111915936+only-dev-time@users.noreply.github.com> Date: Tue, 12 Mar 2024 23:27:44 +0100 Subject: [PATCH 09/10] fix SQL-Statement for pagination --- hive/server/bridge_api/cursor.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hive/server/bridge_api/cursor.py b/hive/server/bridge_api/cursor.py index 91d14d07..317bb5fc 100644 --- a/hive/server/bridge_api/cursor.py +++ b/hive/server/bridge_api/cursor.py @@ -478,10 +478,11 @@ async def pids_by_bookmarks(db, account: str, sort: str = 'bookmarks', category: # order by age of bookmarks order_by = "bookmarks.bookmarked_at DESC" if start_permlink: - join = "JOIN hive_posts AS posts ON bookmarks.post_id = posts.id" seek = """ AND bookmarks.bookmarked_at < ( - SELECT bookmarked_at FROM hive_bookmarks WHERE post_id = :start_id + SELECT bookmarked_at FROM hive_bookmarks + WHERE post_id = :start_id + AND account = :account ) """ From 05a2b27bb955fc50f4abd8f22a83abe278504a43 Mon Sep 17 00:00:00 2001 From: only-dev-time <111915936+only-dev-time@users.noreply.github.com> Date: Fri, 15 Mar 2024 22:45:24 +0100 Subject: [PATCH 10/10] optimize indexes for table hive_bookmarks and other minor fixes --- hive/db/schema.py | 6 +++--- hive/indexer/bookmark.py | 3 --- hive/indexer/custom_op.py | 2 +- hive/server/bridge_api/cursor.py | 3 +-- hive/server/bridge_api/methods.py | 3 ++- 5 files changed, 7 insertions(+), 10 deletions(-) diff --git a/hive/db/schema.py b/hive/db/schema.py index 5dab9cec..537220e9 100644 --- a/hive/db/schema.py +++ b/hive/db/schema.py @@ -364,14 +364,14 @@ def build_metadata_bookmarks(metadata=None): sa.Table( 'hive_bookmarks', metadata, - sa.Column('id', sa.Integer, primary_key=True, autoincrement=True), # TODO notwendig? sa.Column('account', VARCHAR(16), nullable=False), sa.Column('post_id', sa.Integer, nullable=False), sa.Column('bookmarked_at', sa.DateTime, nullable=False), sa.UniqueConstraint('account', 'post_id', name='hive_bookmarks_ux1'), - sa.Index('hive_bookmarks_ix1', 'account'), # bookmarks from account - sa.Index('hive_bookmarks_ix2', 'post_id'), # bookmarks for a post + sa.Index('hive_bookmarks_ix1', 'post_id'), + sa.Index('hive_bookmarks_ix2', 'account', 'post_id'), + sa.Index('hive_bookmarks_ix3', 'account', 'bookmarked_at'), ) return metadata diff --git a/hive/indexer/bookmark.py b/hive/indexer/bookmark.py index 1e16f0f7..283f2eec 100644 --- a/hive/indexer/bookmark.py +++ b/hive/indexer/bookmark.py @@ -26,15 +26,12 @@ def bookmark_op(cls, account, op_json, date): sql = """INSERT INTO hive_bookmarks (account, post_id, bookmarked_at) VALUES (:account, :post_id, :at)""" DB.query(sql, **op) - # TODO notify author of bookmark added - # TODO update bookmarks count on post # perform remove bookmark elif op['action'] == 'remove': sql = """DELETE FROM hive_bookmarks WHERE account = :account AND post_id = :post_id""" DB.query(sql, **op) - # TODO update bookmarks count on post @classmethod def _validated_op(cls, account, op, date): diff --git a/hive/indexer/custom_op.py b/hive/indexer/custom_op.py index 564d6949..6593dd2a 100644 --- a/hive/indexer/custom_op.py +++ b/hive/indexer/custom_op.py @@ -92,7 +92,7 @@ def _process_legacy(cls, account, op_json, block_date): author: {type: 'account'}, permlink: {type: 'permlink'}, action: {type: 'str'}, - category: {type: 'str'}} + category: {type: 'str'}} // category currently unused """ if not isinstance(op_json, list): return diff --git a/hive/server/bridge_api/cursor.py b/hive/server/bridge_api/cursor.py index 317bb5fc..9673d726 100644 --- a/hive/server/bridge_api/cursor.py +++ b/hive/server/bridge_api/cursor.py @@ -469,7 +469,6 @@ async def pids_by_payout(db, account: str, start_author: str = '', async def pids_by_bookmarks(db, account: str, sort: str = 'bookmarks', category: str = '', start_author: str = '', start_permlink: str = '', limit: int = 20): """Get a list of post_ids for an author's bookmarks.""" - account_id = await _get_account_id(db, account) seek = '' join = '' start_id = await _get_post_id(db, start_author, start_permlink) if start_permlink else None @@ -508,7 +507,7 @@ async def pids_by_bookmarks(db, account: str, sort: str = 'bookmarks', category: SELECT bookmarks.post_id FROM hive_bookmarks AS bookmarks %s - WHERE account = :account %s + WHERE bookmarks.account = :account %s ORDER BY %s LIMIT :limit """ % (join, seek, order_by) diff --git a/hive/server/bridge_api/methods.py b/hive/server/bridge_api/methods.py index ae5f2a5b..a5887785 100644 --- a/hive/server/bridge_api/methods.py +++ b/hive/server/bridge_api/methods.py @@ -172,7 +172,8 @@ async def get_bookmarked_posts(context, account, sort='bookmarks', category='', start_author = valid_account(start_author, allow_empty=True) start_permlink = valid_permlink(start_permlink, allow_empty=True) start = (start_author, start_permlink) - limit = valid_limit(limit, 100) + limit = valid_limit(limit, 50) + category = '' # currently unused # check blacklist accounts _id = await db.query_one("SELECT id FROM hive_posts_status WHERE author = :n AND list_type = '3'", n=account)