From 2ebd987f0d002803fa23930c17c8f16f92a5446d Mon Sep 17 00:00:00 2001 From: Durinthal Date: Sun, 15 Jan 2023 17:41:20 -0500 Subject: [PATCH 01/20] Add script to check which users commented on a set of posts --- scripts/participation_check.py | 85 ++++++++++++++++++++++++++++++++++ src/utils/reddit.py | 12 +++++ 2 files changed, 97 insertions(+) create mode 100644 scripts/participation_check.py diff --git a/scripts/participation_check.py b/scripts/participation_check.py new file mode 100644 index 0000000..03cd61b --- /dev/null +++ b/scripts/participation_check.py @@ -0,0 +1,85 @@ +""" +Looks for users that commented in a minimum percentage of threads in the list. Use with -h for instructions. +""" + +import argparse +from collections import Counter +from datetime import timedelta + +from data.post_data import PostModel +from services import post_service, comment_service +from utils import reddit as reddit_utils +from utils.logger import logger + + +def get_post_users(post: PostModel, max_time_after: timedelta): + comments = comment_service.get_comments_by_post_id(post.id36) + usernames = set() + for comment in comments: + # Deleted/unknown users don't count. + if not comment.author: + continue + # Skip comments past the time limit. + if post.created_time + max_time_after < comment.created_time: + continue + usernames.add(comment.author) + + return usernames + + +def load_post_list(post_list: list[str]) -> list[str]: + post_id_list = [] + for post_url in post_list: + parsed_post_id = reddit_utils.POST_ID_REGEX.match(post_url) + if not parsed_post_id: + continue + + post_id_list.append(parsed_post_id.groupdict().get("id")) + return post_id_list + + +def main(post_list: list[str], min_percentage: float, max_time_after: timedelta): + post_id_list = load_post_list(post_list) + post_with_users = {} + for post_id in post_id_list: + post = post_service.get_post_by_id(post_id) + if not post: + logger.warning(f"Post with ID {post_id} not found") + continue + + user_set = get_post_users(post, max_time_after) + user_set.add(post.author) + post_with_users[post] = list(user_set) + + post_count = len(post_with_users) + user_counts = Counter([username for user_list in post_with_users.values() for username in user_list]) + users_sorted = sorted(user_counts.items(), key=lambda kv: kv[1], reverse=True) + for username, user_count in users_sorted: + user_percentage = int(round(user_count / post_count, 2) * 100) + if user_percentage < min_percentage: + break + logger.info(f"{username:>22}: {user_percentage:>3}% ({user_count:3} / {post_count:<3})") + + +def _get_parser() -> argparse.ArgumentParser: + new_parser = argparse.ArgumentParser(description="Check users who commented on a post.") + new_parser.add_argument("-f", "--file", action="store", required=True, help="File path to list of post URLs to check.") + new_parser.add_argument("-p", "--percentage", action="store", type=int, default=80, help="Minimum percentage of threads required (default 80).") + new_parser.add_argument( + "-d", + "--max_days", + type=lambda d: timedelta(days=int(d)), + default=timedelta(days=7), + help="Maximum number of days after thread posting to count comments toward participation (default 7).", + ) + return new_parser + + +if __name__ == "__main__": + parser = _get_parser() + args = parser.parse_args() + + with open(args.file, "r") as post_file: + post_url_list = [s.strip() for s in post_file.readlines()] + + main(post_url_list, args.percentage, args.max_days) diff --git a/src/utils/reddit.py b/src/utils/reddit.py index 81c041c..1903bc1 100644 --- a/src/utils/reddit.py +++ b/src/utils/reddit.py @@ -1,6 +1,7 @@ """Utilities regarding Reddit posts/users/etc""" import copy +import re import typing import mintotp @@ -10,6 +11,17 @@ from data.base_data import BaseModel +POST_ID_REGEX = re.compile(r""" + (?: + (https?://(?:\w+\.)?reddit\.com(?:/r/anime/comments))| # Regular reddit URLs on any subdomain + (https?://redd\.it)| # Shortened URLs + /comments # Relative links + ) + /(?P\w+) # Post ID, the part we care about + (?:/?\w*/?(?:\.compact)?\??)? # Everything afterward is irrelevant +""", re.VERBOSE) + + _b36_alphabet = "0123456789abcdefghijklmnopqrstuvwxyz" From 9c9a41b09dcf6258879ea62801125039fc9470e5 Mon Sep 17 00:00:00 2001 From: Durinthal Date: Sun, 15 Jan 2023 17:42:04 -0500 Subject: [PATCH 02/20] Include bans by bots in monthly total --- scripts/reports.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/reports.py b/scripts/reports.py index 6bc07a5..ceca215 100644 --- a/scripts/reports.py +++ b/scripts/reports.py @@ -94,10 +94,10 @@ def _report_monthly(report_args: argparse.Namespace): ) banned_users = mod_action_service.count_mod_actions( - "banuser", start_date, end_date, distinct=True, exclude_mod_accounts_list=_bots_and_admins + "banuser", start_date, end_date, distinct=True, exclude_mod_accounts_list=mod_constants.ADMINS ) permabanned_users = mod_action_service.count_mod_actions( - "banuser", start_date, end_date, distinct=True, details="permanent", exclude_mod_accounts_list=_bots_and_admins + "banuser", start_date, end_date, distinct=True, details="permanent", exclude_mod_accounts_list=mod_constants.ADMINS ) banned_users_bots = mod_action_service.count_mod_actions( "banuser", start_date, end_date, distinct=True, mod_accounts_list=mod_constants.BOTS From fa969ca97f83e87670f84ce6fd620761850e1e9e Mon Sep 17 00:00:00 2001 From: Durinthal Date: Sun, 15 Jan 2023 17:55:30 -0500 Subject: [PATCH 03/20] Separate CDF activity for mod apps --- scripts/modapps.py | 20 +++++++++++++++----- src/data/comment_data.py | 28 ++++++++++++++++++---------- src/services/comment_service.py | 4 ++-- 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/scripts/modapps.py b/scripts/modapps.py index f27e606..62e847c 100644 --- a/scripts/modapps.py +++ b/scripts/modapps.py @@ -45,9 +45,13 @@ def process_row(row, activity_start_date, activity_end_date): # and overall history on /r/anime. activity_start_time_str = activity_start_date.isoformat() activity_end_time_str = activity_end_date.isoformat() - user_comments_window = len( + user_comments_window_with_cdf = len( comment_service.get_comments_by_username(username, activity_start_time_str, activity_end_time_str) ) + user_comments_window = len( + comment_service.get_comments_by_username(username, activity_start_time_str, activity_end_time_str, True) + ) + cdf_window = user_comments_window_with_cdf - user_comments_window user_posts_window = len( post_service.get_posts_by_username(username, activity_start_time_str, activity_end_time_str) ) @@ -59,14 +63,20 @@ def process_row(row, activity_start_date, activity_end_date): mod_actions[mod_action.action].append(mod_action) mod_actions_str = ", ".join(f"{action} ({len(action_list)})" for action, action_list in mod_actions.items()) - user_comments_total = len(comment_service.get_comments_by_username(username, "2021-06-01")) - user_posts_total = len(post_service.get_posts_by_username(username, "2021-06-01")) + user_comments_total_with_cdf = len(comment_service.get_comments_by_username(username, "2020-01-01")) + user_comments_total = len(comment_service.get_comments_by_username(username, "2020-01-01", exclude_cdf=True)) + cdf_total = user_comments_total_with_cdf - user_comments_total + user_posts_total = len(post_service.get_posts_by_username(username, "2020-01-01")) passes_activity_threshold = "✅" if user_comments_window + user_posts_window > 50 else "❌" response_body += f"### Activity in past 90 days {passes_activity_threshold}\n\n" - response_body += f"> Comments: {user_comments_window} ({user_comments_total} since 2021-06-01)" - response_body += f" Submissions: {user_posts_window} ({user_posts_total} since 2021-06-01)\n\n" + response_body += f"> Comments excluding CDF: {user_comments_window} ({user_comments_total} since 2020-01-01)" + if cdf_window or cdf_total: + response_body += f" (including CDF: {cdf_window}, {cdf_total} since 2020-01-01)" + else: + response_body += f" (no CDF activity)" + response_body += f" Submissions: {user_posts_window} ({user_posts_total} since 2020-01-01)\n\n" response_body += f"> Mod actions since 2021-01-01: {mod_actions_str}\n\n" redditor = reddit.redditor(username) diff --git a/src/data/comment_data.py b/src/data/comment_data.py index becc7d2..253f373 100644 --- a/src/data/comment_data.py +++ b/src/data/comment_data.py @@ -61,27 +61,35 @@ def get_comments_by_post_id(self, post_id: int) -> list[CommentModel]: return [CommentModel(row) for row in result_rows] def get_comments_by_username( - self, username: str, start_date: str = None, end_date: str = None + self, username: str, start_date: str = None, end_date: str = None, exclude_cdf: bool = False ) -> list[CommentModel]: - where_clauses = ["lower(author) = :username"] + where_clauses = ["lower(c.author) = :username"] sql_kwargs = {"username": username.lower()} if start_date: - where_clauses.append("created_time >= :start_date") + where_clauses.append("c.created_time >= :start_date") sql_kwargs["start_date"] = start_date if end_date: - where_clauses.append("created_time < :end_date") + where_clauses.append("c.created_time < :end_date") sql_kwargs["end_date"] = end_date where_str = " AND ".join(where_clauses) - sql = text( - f""" - SELECT * FROM comments - WHERE {where_str}; - """ - ) + if exclude_cdf: + sql = text( + f""" + SELECT * FROM comments c JOIN posts p ON c.post_id = p.id + WHERE {where_str} AND p.title not like 'Casual Discussion Fridays - Week of %' + """ + ) + else: + sql = text( + f""" + SELECT * FROM comments c + WHERE {where_str}; + """ + ) result_rows = self.execute(sql, **sql_kwargs) return [CommentModel(row) for row in result_rows] diff --git a/src/services/comment_service.py b/src/services/comment_service.py index d55a419..93a4f27 100644 --- a/src/services/comment_service.py +++ b/src/services/comment_service.py @@ -33,12 +33,12 @@ def get_comments_by_post_id(post_id: Union[str, int]) -> list[CommentModel]: return _comment_data.get_comments_by_post_id(post_id) -def get_comments_by_username(username: str, start_date: str = None, end_date: str = None) -> list[CommentModel]: +def get_comments_by_username(username: str, start_date: str = None, end_date: str = None, exclude_cdf: bool = False) -> list[CommentModel]: """ Gets all comments by a user, optionally within a specified time frame. """ - return _comment_data.get_comments_by_username(username, start_date, end_date) + return _comment_data.get_comments_by_username(username, start_date, end_date, exclude_cdf) def count_comments(start_date: date = None, end_date: date = None, exclude_authors: list = None) -> int: From aaaf179d2cf86a5b8f92dc780f0fe69e59447f1d Mon Sep 17 00:00:00 2001 From: Durinthal Date: Sun, 15 Jan 2023 17:56:06 -0500 Subject: [PATCH 04/20] Include deleted time for post embeds when available --- src/services/post_service.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/services/post_service.py b/src/services/post_service.py index 1c35090..4aa38b8 100644 --- a/src/services/post_service.py +++ b/src/services/post_service.py @@ -134,6 +134,10 @@ def format_post_embed(post: PostModel): if post.metadata and post.metadata.get("spoiler"): embed_json["fields"].append({"name": "Spoiler", "value": "\u200b", "inline": True}) + if post.deleted_time: + deleted_timestamp = int(post.deleted_time.timestamp()) + embed_json["fields"].append({"name": "Deleted", "value": f"", "inline": True}) + return embed_json From 143868ddabbce419db16b6b06634483ae283196d Mon Sep 17 00:00:00 2001 From: Durinthal Date: Sun, 15 Jan 2023 18:03:39 -0500 Subject: [PATCH 05/20] Include post author in mod log Discord message --- src/feeds/mod_log.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/feeds/mod_log.py b/src/feeds/mod_log.py index 345b848..3b10c82 100644 --- a/src/feeds/mod_log.py +++ b/src/feeds/mod_log.py @@ -244,7 +244,9 @@ def send_discord_message(mod_action: ModActionModel): embed_json["title"] = f"{mod_action.mod}: {mod_action.action} by {mod_action.target_user}" elif mod_action.target_post_id: target = post_service.get_post_by_id(mod_action.target_post_id) - title = discord.escape_formatting(f"{mod_action.mod}: {mod_action.action} - {target.title}") + title = discord.escape_formatting( + f"{mod_action.mod}: {mod_action.action} - {target.title} by {mod_action.target_user}" + ) embed_json["title"] = title[:253] + "..." if len(title) > 256 else title elif mod_action.target_user: target = user_service.get_user(mod_action.target_user) From e9157945ba3f86c97aee6eb9e11069b022de32fb Mon Sep 17 00:00:00 2001 From: Durinthal Date: Sun, 15 Jan 2023 18:04:10 -0500 Subject: [PATCH 06/20] Linting fixes --- scripts/participation_check.py | 13 +++++++++++-- scripts/reports.py | 7 ++++++- src/services/comment_service.py | 4 +++- src/utils/reddit.py | 7 +++++-- 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/scripts/participation_check.py b/scripts/participation_check.py index 03cd61b..ff78dc8 100644 --- a/scripts/participation_check.py +++ b/scripts/participation_check.py @@ -63,8 +63,17 @@ def main(post_list: list[str], min_percentage: float, max_time_after: timedelta) def _get_parser() -> argparse.ArgumentParser: new_parser = argparse.ArgumentParser(description="Check users who commented on a post.") - new_parser.add_argument("-f", "--file", action="store", required=True, help="File path to list of post URLs to check.") - new_parser.add_argument("-p", "--percentage", action="store", type=int, default=80, help="Minimum percentage of threads required (default 80).") + new_parser.add_argument( + "-f", "--file", action="store", required=True, help="File path to list of post URLs to check." + ) + new_parser.add_argument( + "-p", + "--percentage", + action="store", + type=int, + default=80, + help="Minimum percentage of threads required (default 80).", + ) new_parser.add_argument( "-d", "--max_days", diff --git a/scripts/reports.py b/scripts/reports.py index ceca215..6bcb52c 100644 --- a/scripts/reports.py +++ b/scripts/reports.py @@ -97,7 +97,12 @@ def _report_monthly(report_args: argparse.Namespace): "banuser", start_date, end_date, distinct=True, exclude_mod_accounts_list=mod_constants.ADMINS ) permabanned_users = mod_action_service.count_mod_actions( - "banuser", start_date, end_date, distinct=True, details="permanent", exclude_mod_accounts_list=mod_constants.ADMINS + "banuser", + start_date, + end_date, + distinct=True, + details="permanent", + exclude_mod_accounts_list=mod_constants.ADMINS, ) banned_users_bots = mod_action_service.count_mod_actions( "banuser", start_date, end_date, distinct=True, mod_accounts_list=mod_constants.BOTS diff --git a/src/services/comment_service.py b/src/services/comment_service.py index 93a4f27..1ffda2a 100644 --- a/src/services/comment_service.py +++ b/src/services/comment_service.py @@ -33,7 +33,9 @@ def get_comments_by_post_id(post_id: Union[str, int]) -> list[CommentModel]: return _comment_data.get_comments_by_post_id(post_id) -def get_comments_by_username(username: str, start_date: str = None, end_date: str = None, exclude_cdf: bool = False) -> list[CommentModel]: +def get_comments_by_username( + username: str, start_date: str = None, end_date: str = None, exclude_cdf: bool = False +) -> list[CommentModel]: """ Gets all comments by a user, optionally within a specified time frame. """ diff --git a/src/utils/reddit.py b/src/utils/reddit.py index 1903bc1..ef65b4f 100644 --- a/src/utils/reddit.py +++ b/src/utils/reddit.py @@ -11,7 +11,8 @@ from data.base_data import BaseModel -POST_ID_REGEX = re.compile(r""" +POST_ID_REGEX = re.compile( + r""" (?: (https?://(?:\w+\.)?reddit\.com(?:/r/anime/comments))| # Regular reddit URLs on any subdomain (https?://redd\.it)| # Shortened URLs @@ -19,7 +20,9 @@ ) /(?P\w+) # Post ID, the part we care about (?:/?\w*/?(?:\.compact)?\??)? # Everything afterward is irrelevant -""", re.VERBOSE) +""", + re.VERBOSE, +) _b36_alphabet = "0123456789abcdefghijklmnopqrstuvwxyz" From a71054b5dc2cd8a93911ed91e48bb0e4aa52ff42 Mon Sep 17 00:00:00 2001 From: Durinthal Date: Sun, 15 Jan 2023 18:07:09 -0500 Subject: [PATCH 07/20] Linting fixes --- scripts/modapps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/modapps.py b/scripts/modapps.py index 62e847c..51e2712 100644 --- a/scripts/modapps.py +++ b/scripts/modapps.py @@ -75,7 +75,7 @@ def process_row(row, activity_start_date, activity_end_date): if cdf_window or cdf_total: response_body += f" (including CDF: {cdf_window}, {cdf_total} since 2020-01-01)" else: - response_body += f" (no CDF activity)" + response_body += " (no CDF activity)" response_body += f" Submissions: {user_posts_window} ({user_posts_total} since 2020-01-01)\n\n" response_body += f"> Mod actions since 2021-01-01: {mod_actions_str}\n\n" From e526a0f602341284db764e7a33751a7c085ec081 Mon Sep 17 00:00:00 2001 From: Durinthal Date: Tue, 17 Jan 2023 22:02:44 -0500 Subject: [PATCH 08/20] Expand padding in front page embed table for 7-character post IDs --- scripts/frontpage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/frontpage.py b/scripts/frontpage.py index 875e135..321791d 100644 --- a/scripts/frontpage.py +++ b/scripts/frontpage.py @@ -31,7 +31,7 @@ def _format_line(submission, position, rank_change, total_hours): line += f" {submission.score:>5}" - line += " {:>24}".format(f"[{submission.link_flair_text}]({submission.id})") + line += " {:>25}".format(f"[{submission.link_flair_text}]({submission.id})") line += f" <{submission.author.name}>" From c3489207841aa21c703810d3c561051d03fddb82 Mon Sep 17 00:00:00 2001 From: Durinthal Date: Sat, 17 Jun 2023 14:22:21 -0400 Subject: [PATCH 09/20] Add notification for uncommon settings changes --- src/feeds/mod_log.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/feeds/mod_log.py b/src/feeds/mod_log.py index 3b10c82..ed4643d 100644 --- a/src/feeds/mod_log.py +++ b/src/feeds/mod_log.py @@ -75,6 +75,14 @@ def _format_action_embed_field(mod_action_model: ModActionModel = None) -> Optio if mod_action.action in mod_constants.MOD_ACTIONS_ALWAYS_NOTIFY: send_notification = True + if mod_action.action == "editsettings" and mod_action.details not in ( + "description", + "del_image", + "upload_image", + "header_title", + ): + send_notification = True + if mod_action.mod.name not in active_mods: # Add them to the database if necessary. mod_user = user_service.get_user(mod_action.mod.name) From aff880a56546c3fac20df5cf88933372966c54c2 Mon Sep 17 00:00:00 2001 From: Durinthal Date: Sat, 17 Jun 2023 14:31:09 -0400 Subject: [PATCH 10/20] Black fixes --- scripts/frontpage_sqlite_migration.py | 1 - src/data/base_data.py | 1 - src/data/mod_action_data.py | 1 - src/feeds/sub_mentions.py | 1 - src/utils/logger.py | 1 - 5 files changed, 5 deletions(-) diff --git a/scripts/frontpage_sqlite_migration.py b/scripts/frontpage_sqlite_migration.py index 68301f0..401ef9d 100644 --- a/scripts/frontpage_sqlite_migration.py +++ b/scripts/frontpage_sqlite_migration.py @@ -66,7 +66,6 @@ def migrate_posts(offset=0): def migrate_snapshots(date, hour): - conn = sqlite3.connect(DB_FILE) conn.row_factory = sqlite3.Row diff --git a/src/data/base_data.py b/src/data/base_data.py index 1134414..7733a1a 100644 --- a/src/data/base_data.py +++ b/src/data/base_data.py @@ -126,7 +126,6 @@ def insert(self, model: BaseModel, error_on_conflict: bool = True): return new_model def update(self, model: BaseModel): - if model.pk_field in model.modified_fields: raise NotImplementedError(f"Can't update the primary key of model {model}!") diff --git a/src/data/mod_action_data.py b/src/data/mod_action_data.py index b5c65d5..b1dfb70 100644 --- a/src/data/mod_action_data.py +++ b/src/data/mod_action_data.py @@ -39,7 +39,6 @@ def get_mod_action_by_id(self, mod_action_id: str) -> Optional[ModActionModel]: def get_mod_actions_targeting_post( self, post_id: int, actions: list[str] = None, limit: int = None, order: str = "DESC" ): - where_clauses = ["target_post_id = :post_id"] sql_kwargs = {"post_id": post_id} diff --git a/src/feeds/sub_mentions.py b/src/feeds/sub_mentions.py index 4f25145..84fece9 100644 --- a/src/feeds/sub_mentions.py +++ b/src/feeds/sub_mentions.py @@ -9,7 +9,6 @@ def check_inbox(reddit): - for message in reddit.inbox.unread(limit=5): if message.author != "Sub_Mentions": message.mark_read() diff --git a/src/utils/logger.py b/src/utils/logger.py index 6557053..d09c7bb 100644 --- a/src/utils/logger.py +++ b/src/utils/logger.py @@ -9,7 +9,6 @@ def _setup_logging(): - global logger logger = logging.getLogger("modbot") From d230460bb7835113834e9709f6642eeffdfccd87 Mon Sep 17 00:00:00 2001 From: Durinthal Date: Sat, 17 Jun 2023 14:57:44 -0400 Subject: [PATCH 11/20] Fix spam feed --- src/feeds/consolidated.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/feeds/consolidated.py b/src/feeds/consolidated.py index c0b7a9a..9b40f5c 100644 --- a/src/feeds/consolidated.py +++ b/src/feeds/consolidated.py @@ -23,7 +23,6 @@ def monitor_streams(posts: bool = False, comments: bool = False, log: bool = Fal submission_stream = None comment_stream = None mod_log_stream = None - spam_stream = None while True: try: @@ -42,9 +41,6 @@ def monitor_streams(posts: bool = False, comments: bool = False, log: bool = Fal if log: logger.info("Initializing mod log stream...") mod_log_stream = subreddit.mod.stream.log(skip_existing=False, pause_after=-1) - if spam: - logger.info("Initializing spam stream...") - spam_stream = subreddit.mod.stream.spam(skip_existing=False, pause_after=-1) while True: if log: @@ -73,7 +69,7 @@ def monitor_streams(posts: bool = False, comments: bool = False, log: bool = Fal if spam: logger.debug("Starting spam stream...") - for item in spam_stream: + for item in subreddit.mod.spam(): if item is None: time.sleep(3) break From 3346cb35c374718acddacd8b8a968c0d53addba7 Mon Sep 17 00:00:00 2001 From: Durinthal Date: Sat, 1 Jul 2023 10:02:54 -0400 Subject: [PATCH 12/20] Add notifications to unspam script --- scripts/unspam.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/scripts/unspam.py b/scripts/unspam.py index 950407a..d9a3d0b 100644 --- a/scripts/unspam.py +++ b/scripts/unspam.py @@ -5,11 +5,11 @@ """ import argparse -from datetime import datetime +from datetime import datetime, timezone import config_loader from services import post_service, comment_service -from utils import reddit as reddit_utils +from utils import discord, reddit as reddit_utils from utils.logger import logger @@ -28,6 +28,18 @@ def approve_user_items(username, start_date, end_date): skip_list = [] logger.info(f"Found {len(id_list)} items in database by {username}") + embed_json = { + "author": { + "name": f"Unspam - /u/{username}", + }, + "title": f"Beginning to unspam /u/{username}", + "description": f"Found {len(id_list)} items in the database", + "timestamp": datetime.now(tz=timezone.utc).isoformat(), + "url": f"https://reddit.com/user/{username}", + "fields": [], + "color": 0x00CC00, + } + discord.send_webhook_message(config_loader.DISCORD["webhook_url"], {"embeds": [embed_json]}) for item in reddit.info(fullnames=id_list): try: @@ -43,6 +55,18 @@ def approve_user_items(username, start_date, end_date): logger.info(f"Finished unspamming {username}") logger.info(f"Total {len(approve_list)} approved, {len(skip_list)} skipped, {len(error_list)} errors.") + embed_json = { + "author": { + "name": f"Unspam - /u/{username}", + }, + "title": f"Finished unspamming /u/{username}", + "description": f"{len(approve_list)} approved, {len(skip_list)} skipped, {len(error_list)} errors", + "timestamp": datetime.now(tz=timezone.utc).isoformat(), + "url": f"https://reddit.com/user/{username}", + "fields": [], + "color": 0x00CC00, + } + discord.send_webhook_message(config_loader.DISCORD["webhook_url"], {"embeds": [embed_json]}) def _get_parser() -> argparse.ArgumentParser: From 3cce99b8327339d1903bcd71f6985143c029ad20 Mon Sep 17 00:00:00 2001 From: Durinthal Date: Thu, 27 Jul 2023 20:52:03 -0400 Subject: [PATCH 13/20] Update sub mention feed with new tool format --- src/feeds/sub_mentions.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/feeds/sub_mentions.py b/src/feeds/sub_mentions.py index 84fece9..fa39910 100644 --- a/src/feeds/sub_mentions.py +++ b/src/feeds/sub_mentions.py @@ -1,3 +1,4 @@ +import re import time import config_loader @@ -10,20 +11,30 @@ def check_inbox(reddit): for message in reddit.inbox.unread(limit=5): - if message.author != "Sub_Mentions": + logger.info(f"Checking message from {message.author}") + if message.author != "Sub_Mentions" and not message.author.name.startswith("feedcomber-"): message.mark_read() continue - author, desc = message.body.split("\n\n", 1) + message_body = re.sub("\n\n___\n\n", "\n\n____\n\n", message.body) # standardize template + header, body = message_body.split("\n\n____\n\n", 1) + body, footer = body.rsplit("\n\n____\n\n", 1) + header_parts = header.split("\n\n") + link = header_parts[-1] + author = header_parts[-2] - title = message.subject.replace("[Notification] Your subreddit has been mentioned in ", "") - title = title.replace("!", " - ") + # Check to see that it's actually a reference to /r/anime and not something else. + if not re.search(r"\br/anime\b", body): + message.mark_read() + continue + + title = re.sub(r"^.*/(r/\w+)!?$", r"/\1 - ", message.subject) title += author.replace("Author: ", "") logger.info(f"Processing message {title}") - desc = desc[:-279] # removes info at the end of the message - desc = desc.replace("(/r/", "(https://www.reddit.com/r/") # hyperlinks reddit links - desc = desc.replace("\n___\n", "") + desc = link.replace("(/r/", "(https://www.reddit.com/r/") # hyperlink to reference + desc += "\n\n" + body + if len(desc) >= 2000: # message length (max for webhook is 2000) desc = desc[:1997] + "..." From 59f90f79f2865279348e2a6a9229fbd0a877c755 Mon Sep 17 00:00:00 2001 From: Durinthal Date: Thu, 27 Jul 2023 21:03:04 -0400 Subject: [PATCH 14/20] Add check to sub mention feed for redesign formatting of underscores --- src/feeds/sub_mentions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/feeds/sub_mentions.py b/src/feeds/sub_mentions.py index fa39910..df7194f 100644 --- a/src/feeds/sub_mentions.py +++ b/src/feeds/sub_mentions.py @@ -24,7 +24,8 @@ def check_inbox(reddit): author = header_parts[-2] # Check to see that it's actually a reference to /r/anime and not something else. - if not re.search(r"\br/anime\b", body): + # Negative lookahead is to catch weird formatting from redesign handling underscores, e.g. r/anime\_irl + if not re.search(r"\br/anime\b(?!\\?_)", body): message.mark_read() continue From 9e02fdcfb02902273f651a08753001ad7bc45f39 Mon Sep 17 00:00:00 2001 From: Durinthal Date: Thu, 27 Jul 2023 21:11:14 -0400 Subject: [PATCH 15/20] Add check to sub mention feed for misuse of hyphen instead of underscore --- src/feeds/sub_mentions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/feeds/sub_mentions.py b/src/feeds/sub_mentions.py index df7194f..26c20bb 100644 --- a/src/feeds/sub_mentions.py +++ b/src/feeds/sub_mentions.py @@ -25,7 +25,8 @@ def check_inbox(reddit): # Check to see that it's actually a reference to /r/anime and not something else. # Negative lookahead is to catch weird formatting from redesign handling underscores, e.g. r/anime\_irl - if not re.search(r"\br/anime\b(?!\\?_)", body): + # or people using a hyphen instead of underscore like r/anime-titties + if not re.search(r"\br/anime\b(?!(\\?_)|-)", body): message.mark_read() continue From 0d09423ea8f4f27d81c3583421a9fbf5f87074fa Mon Sep 17 00:00:00 2001 From: Durinthal Date: Fri, 18 Aug 2023 23:03:50 -0400 Subject: [PATCH 16/20] Update praw version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fd89041..aa8a265 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -praw==7.6.0 +praw==7.7.1 psycopg2-binary==2.9.3 requests==2.27.1 SQLAlchemy==1.4.25 From ae24d65625e93eb94d5b9b68f9b2f81fd6b54b7b Mon Sep 17 00:00:00 2001 From: Durinthal Date: Fri, 18 Aug 2023 23:05:29 -0400 Subject: [PATCH 17/20] Migrate weeklies subreddit updater scripts from other repo https://github.com/r-anime/weeklies/ --- src/old/cdf.py | 85 ++++++++++++++++++ src/old/daily.py | 92 ++++++++++++++++++++ src/old/disc.py | 3 + src/old/menuupdater.py | 192 +++++++++++++++++++++++++++++++++++++++++ src/old/meta.py | 3 + 5 files changed, 375 insertions(+) create mode 100644 src/old/cdf.py create mode 100644 src/old/daily.py create mode 100644 src/old/disc.py create mode 100644 src/old/menuupdater.py create mode 100644 src/old/meta.py diff --git a/src/old/cdf.py b/src/old/cdf.py new file mode 100644 index 0000000..4de0aa0 --- /dev/null +++ b/src/old/cdf.py @@ -0,0 +1,85 @@ +from datetime import datetime, timezone, timedelta +import sys +import time + +import config_loader +from old.menuupdater import SubredditMenuUpdater +from utils import reddit as reddit_utils +from utils.logger import logger + + +name = "Casual Discussion Fridays" +short_name = "Casual Disc Fridays" +author = "AutoModerator" + +# First wait for new thread to go up and update links (standard) +SubredditMenuUpdater(name=name, short_name=short_name, author=author) + +# Then the CDF-specific stuff +reddit = reddit_utils.get_reddit_instance(config_loader.REDDIT["auth"]) +subreddit = reddit.subreddit(config_loader.REDDIT["subreddit"]) + +# Step 0: get new and old CDF +search_str = f"{name.lower()} author:{author}" +logger.debug(f"Search query: {search_str}") +cdfs = subreddit.search(search_str, sort="new") + +while True: + cdf = next(cdfs) + created_ts = datetime.fromtimestamp(cdf.created_utc, timezone.utc) + if created_ts > datetime.now(timezone.utc) - timedelta(days=1): # today + new_cdf = cdf + elif created_ts < datetime.now(timezone.utc) - timedelta(days=6): # last week + old_cdf = cdf + break + +logger.debug(f'Found new CDF id {new_cdf.id} "{new_cdf.title}"') +logger.debug(f'Found old CDF id {old_cdf.id} "{old_cdf.title}"') + + +# Step 1: Notify old CDF that the new CDF is up +notify_comment = old_cdf.reply( + f""" +Hello CDF users! Since it is Friday, the new CDF is now live. Please follow +[this link]({new_cdf.permalink}) to move on to the new thread. + +[](#heartbot "And don't forget to be nice to new users!") + +A quick note: this thread will remain open for one hour so that you can finish +your conversations. Please **do not** use this thread for spamming or other +undesirable behavior. Excessive violations will result in sanctions. +""" +) +notify_comment.disable_inbox_replies() +notify_comment.mod.distinguish() + +# Step 1.5 Sort new CDF by new +logger.debug("Setting new CDF thread sorting to 'new'") +logger.info(f"Posted notify comment {notify_comment.id} in old CDF") +new_cdf.mod.suggested_sort(sort="new") + +# Step 2: Lock old CDF +logger.debug("Going to sleep for 3600 seconds...") +sys.stdout.flush() +time.sleep(3600) +logger.debug("Waking up. Locking old CDF") + +old_cdf.mod.lock() +logger.info("Old CDF thread has been locked") +last_comment = old_cdf.reply( + f""" +This thread has been locked. +We will see you all in the new Casual Discussion Fridays thread, +which you can find [here]({new_cdf.permalink}). + +Reminder to keep the new discussion *welcoming* and be mindful of new users. +Don't take the shitpost too far — but have fun! + +[](#bot-chan) +""" +) +last_comment.disable_inbox_replies() +last_comment.mod.distinguish(sticky=True) + +logger.debug(f"Last comment {last_comment.id} posted") +logger.info("CDF job complete.") diff --git a/src/old/daily.py b/src/old/daily.py new file mode 100644 index 0000000..87b8956 --- /dev/null +++ b/src/old/daily.py @@ -0,0 +1,92 @@ +from datetime import datetime, timezone, timedelta +import re + +import prawcore.exceptions + +import config_loader +from old.menuupdater import SubredditMenuUpdater +from utils import reddit as reddit_utils +from utils.logger import logger + + +name = "Anime Questions, Recommendations, and Discussion" +short_name = "Daily Megathread" +author = "AnimeMod" + +# First wait for new thread to go up and update links (standard) +SubredditMenuUpdater(name=name, short_name=short_name, author=author) + +# Daily Thread Specific Stuff +reddit = reddit_utils.get_reddit_instance(config_loader.REDDIT["auth"]) +subreddit = reddit.subreddit(config_loader.REDDIT["subreddit"]) + +# Step 0: get new and old Daily +search_str = f'title:"{name.lower()}" author:{author}' +logger.debug(f"Search query: {search_str}") +threads = subreddit.search(search_str, sort="new") + +while True: + thread = next(threads) + created_ts = datetime.fromtimestamp(thread.created_utc, timezone.utc) + if created_ts > datetime.now(timezone.utc) - timedelta(hours=23): # today + new_daily = thread + elif datetime.now(timezone.utc) - timedelta(days=2) < created_ts < datetime.now(timezone.utc) - timedelta(days=1): + old_daily = thread + break + +logger.debug(f'Found new daily id {new_daily.id} "{new_daily.title}"') +logger.debug(f'Found old daily id {old_daily.id} "{old_daily.title}"') + + +# Step 1: Notify old daily that the new daily is up +notify_comment = old_daily.reply( + f""" +Hello /r/anime, a new daily thread has been posted! Please follow +[this link]({new_daily.permalink}) to move on to the new thread +or [search for the latest thread](/r/{subreddit}/search?q=flair%3ADaily&restrict_sr=on&sort=new). + +[](#heartbot "And don't forget to be nice to new users!") +""" +) +notify_comment.disable_inbox_replies() +notify_comment.mod.distinguish(sticky=True) + +logger.debug(f"Posted notify comment {notify_comment.id} in old daily") + +# Step 2: Update old daily body with link to new one +original_text = old_daily.selftext +updated_text = re.sub(r"\[Next Thread »]\(.*?\)", f"[Next Thread »]({new_daily.permalink})", original_text) +# Keep redesign/mobile image embed after edit, e.g. ![img](vu9tn0wcvwka1 "This is the place!") +updated_text = re.sub(r"\[(.*?)]\(https://preview\.redd\.it/(\w+)\..*?\)", r'![img](\g<2> "\g<1>")', updated_text) +old_daily.edit(body=updated_text) + +logger.debug("Updated old daily body with link to new") + +# Step 3: Add sticky comment for the new thread (if it exists) +sticky_comment_wiki = subreddit.wiki["daily_thread/sticky_comment"] +try: + sticky_comment_text = sticky_comment_wiki.content_md.strip() +except prawcore.exceptions.NotFound: + sticky_comment_text = "" + +if sticky_comment_text: + new_sticky_comment = new_daily.reply(sticky_comment_text) + new_sticky_comment.mod.distinguish(sticky=True) + logger.debug("Posted sticky comment to new thread") +else: + logger.debug("No sticky comment for new thread") + +# Step 4: Rewrite links +original_text = new_daily.selftext +# Change redd.it/ links to relative /comments/ +updated_text = re.sub(r"https?://(?:www\.)?redd\.it/(\w+)/?", r"/comments/\g<1>", original_text) +# Keep redesign/mobile image embed after edit +updated_text = re.sub(r"\[(.*?)]\(https://preview\.redd\.it/(\w+)\..*?\)", r'![img](\g<2> "\g<1>")', updated_text) +new_daily.edit(body=updated_text) + +logger.debug("Updated new daily body with relative links to posts") + +# Step 5: Sort by new (since it's broken on reddit's end right now) +new_daily.mod.suggested_sort(sort="new") + +logger.info("Daily thread job complete.") diff --git a/src/old/disc.py b/src/old/disc.py new file mode 100644 index 0000000..251138a --- /dev/null +++ b/src/old/disc.py @@ -0,0 +1,3 @@ +from old.menuupdater import SubredditMenuUpdater + +SubredditMenuUpdater(name="Anime of the Week", short_name="Anime of the Week", author="AnimeMod") diff --git a/src/old/menuupdater.py b/src/old/menuupdater.py new file mode 100644 index 0000000..61c0799 --- /dev/null +++ b/src/old/menuupdater.py @@ -0,0 +1,192 @@ +from datetime import datetime, timezone, timedelta +import re +import time + +import praw + +import config_loader +from utils import reddit +from utils.logger import logger + + +SEARCH_TIMEOUT = 3600 +LINK_REGEX = r"\[{name}\]\(.*?\)" +LINK_FORMAT = "[{name}]({link})" + + +class SubredditMenuUpdater: + def __init__(self, name, short_name, author, debug=False): + """ + Update the subreddit menu to the most recent post with + Used to replace links for weekly megathreads + + This script is supposed to run *at the same time* as the thread to + update is posted. A timeout guarantees that if the post is not found + soon, the script will stop with failure. + + The Reddit mod account, subreddit, and various script settings are + configured in a static file (default `config.ini`). + + :param name: name of the post as written in the title and menu + :param short_name: name to use in redesign topbar (max 20 characters) + :param author: account from which the post was submitted + :param debug: if True, no change will be made to the subreddit + """ + + logger.info(f"Started running subreddit menu updater for {name}") + self.debug = debug + self.reddit = reddit.get_reddit_instance(config_loader.REDDIT["auth"]) + self.subreddit = self.reddit.subreddit(config_loader.REDDIT["subreddit"]) + + post = self._find_post(name, author) + self._update_menus(name, post) + self._update_redesign_menus(name, short_name, post) + logger.info(f"Completed running subreddit menu updater for {name}") + + def _find_post(self, name, author): + search_str = f'title:"{name}" author:{author}'.lower() + search_start_time = time.time() + + logger.debug(f"Started search with query '{search_str}'") + while True: + post = next(self.subreddit.search(search_str, sort="new"), None) + if post is not None: + post_timestamp = datetime.fromtimestamp(post.created_utc, timezone.utc) + if post_timestamp > datetime.now(timezone.utc) - timedelta(days=6): + # guarantees that the post found was created in the past week + logger.debug(f"Post found {post.permalink}") + return post + + if time.time() - search_start_time > SEARCH_TIMEOUT: + raise TimeoutError("Post not found") + time.sleep(15) + + def _update_menus(self, name, post): + """ + Updates the sidebar text by replacing links to `name` with a permalink + to `post`. Links formatting is defined by config. + + For example, the default format for links is `'[{name}](.*)'` + + :param name: name of the link to format + :param post: post which should be linked to + """ + logger.debug("Updating menus on old Reddit") + + pattern_match = LINK_REGEX.format(name=name) + pattern_replace = LINK_FORMAT.format(name=name, link=post.shortlink) + + sidebar = self.subreddit.wiki["config/sidebar"] + sidebar_text = sidebar.content_md + sidebar_updated_text = self._replace_text(pattern_match, pattern_replace, sidebar_text) + + if sidebar_updated_text is None: + logger.debug("No change necessary") + elif self.debug: + logger.debug("Running in debug mode, no change was made to sidebar") + else: + sidebar.edit(content=sidebar_updated_text, reason=f"Changed link for {name}") + logger.debug("Changes saved to sidebar") + + def _update_redesign_menus(self, name, short_name, post): + """ + Updates the menu and widget text on Redesign by replacing links to + `name` with a permaling to `post`. Links formatting is identical to the + formatting used on old Reddit. + + :param name: name of the link to format + :param short_name: name of the link to use in topbar menu (max 20 characters) + :param post: post which should be linked to + """ + logger.debug("Updating menus on Redesign") + + assert len(short_name) <= 20 + + topmenu = self._get_updated_redesign_topmenu(short_name, post.shortlink) + if topmenu is None: + logger.debug("Error updating topmenu") + elif self.debug: + logger.debug("Running in debug mode, no change was made to top menu") + else: + topmenu.mod.update(data=list(topmenu)) + logger.debug("Topbar menu updated") + + pattern_match = LINK_REGEX.format(name=name) + pattern_replace = LINK_FORMAT.format(name=name, link=post.shortlink) + + sidemenu = self._get_redesign_sidemenu(name) + sidemenu_text = sidemenu.text + sidemenu_updated_text = self._replace_text(pattern_match, pattern_replace, sidemenu_text) + + if sidemenu_updated_text is None: + logger.debug("No change necessary") + elif self.debug: + logger.debug("Running in debug mode, no change was made to side menu") + else: + sidemenu.mod.update(text=sidemenu_updated_text) + logger.debug("Sidebar widget updated") + + def _get_updated_redesign_topmenu(self, name, new_url): + """ + Update the menu by replacing links labeled `name` with `new_url` and + return the updated menu. Updates are *not* reflected to the subreddit + by calling this method. + + :param name: text of the menulink to update + :param new_url: replacement url + """ + menu = self.subreddit.widgets.topbar[0] + assert isinstance(menu, praw.models.Menu) + + for item in menu: + if isinstance(item, praw.models.MenuLink): + if item.text == name: + logger.debug(f"Found replaceable MenuLink: {item.text}") + item.url = new_url + elif isinstance(item, praw.models.Submenu): + for subitem in item: + if isinstance(subitem, praw.models.MenuLink): + if subitem.text == name: + logger.debug(f"Found replaceable MenuLink: {item.text}") + subitem.url = new_url + else: + logger.debug(f"Wrong type found searching for MenuLink: {item.__class__}") + else: + logger.debug(f"Wrong type found searching for MenuLink: {item.__class__}") + + return menu + + def _get_redesign_sidemenu(self, name): + """ + Return the sidebar widget containing a link to `name`. + + :param name: name of the link to update + """ + sidebar = self.subreddit.widgets.sidebar + + pattern_match = LINK_REGEX.format(name=name) + + for widget in sidebar: + if isinstance(widget, praw.models.TextArea): + matches = re.findall(pattern_match, widget.text) + if matches: + logger.debug(f"Found matching side widget '{widget.shortName}'") + return widget + + logger.debug("Found no sidebar widget with replaceable match") + return None + + def _replace_text(self, pattern_match, pattern_replace, text): + matches = re.findall(pattern_match, text) + if not matches: + logger.debug("Found no replaceable match") + return None + + logger.debug( + "Found replaceable matches\n" + + f'\t\t\tOld text: {" // ".join(matches)}\n' + + f"\t\t\tNew text: {pattern_replace}" + ) + + text_replaced = re.sub(pattern_match, pattern_replace, text) + return text_replaced diff --git a/src/old/meta.py b/src/old/meta.py new file mode 100644 index 0000000..182bd15 --- /dev/null +++ b/src/old/meta.py @@ -0,0 +1,3 @@ +from old.menuupdater import SubredditMenuUpdater + +SubredditMenuUpdater(name="Meta Thread", short_name="Monthly Meta Thread", author="AnimeMod") From ecbe31da08655c1fd583adeb21072d005e085612 Mon Sep 17 00:00:00 2001 From: Durinthal Date: Sat, 19 Aug 2023 17:42:11 -0400 Subject: [PATCH 18/20] Migrate selfpromobot from older repo https://github.com/r-anime/selfpromobot --- src/old/selfpromobot.py | 220 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100755 src/old/selfpromobot.py diff --git a/src/old/selfpromobot.py b/src/old/selfpromobot.py new file mode 100755 index 0000000..446b0ad --- /dev/null +++ b/src/old/selfpromobot.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 + +import time +from datetime import datetime, timezone, timedelta + +import config_loader +from utils import reddit as reddit_utils +from utils.logger import logger + +global DEBUG + +REMOVAL_MESSAGE_TEMPLATE = """Sorry, your submission has been removed.\n\n{message}\n\n +*I am a bot, and this action was performed automatically. Please +[contact the moderators of this subreddit](https://www.reddit.com/message/compose/?to=/r/anime) +if you have any questions or concerns.*""" + +# Number of posts to check on each run +posts_per_run = 25 +# Maximum number of items to check in a user's history +history = 500 +# Interval between multiple checks +interval = 60 + + +def main(subreddit): + """ + Main loop. + + :param subreddit: the praw Subreddit instance + """ + + # Only check posts once + checked = list() + + if DEBUG: + logger.warning("Running in debug mode") + + while True: + logger.info(f"Checking for the {posts_per_run} most recent posts") + for post in subreddit.new(limit=posts_per_run): + if post not in checked: + # Note : only the first violation will be reported + # Check fanart frequency + if is_fanart(post): + logger.debug(f"Found fanart {post} by {post.author.name}") + check_fanart_frequency(post) + # Check clip frequency + if is_clip(post): + logger.debug(f"Found clip {post} by {post.author.name}") + check_clip_frequency(post) + # Check video edit frequency + if is_video_edit(post): + logger.debug(f"Found video edit {post} by {post.author.name}") + check_video_edit_frequency(post) + # Check video frequency + if is_video(post): + logger.debug(f"Found video {post} by {post.author.name}") + check_video_frequency(post) + checked.append(post) + + # Only remember the most recent posts, as the others won't flow back into /new + checked = checked[-3 * posts_per_run :] # noqa: E203 + + time.sleep(interval) + + +def remove(post, reason, message=None): + if DEBUG: + logger.info(f" !-> Not removing {post} by {post.author.name} in debug mode") + else: + logger.info(f" --> Removing post {post} by {post.author.name}") + if is_removed(post): + logger.warning(" !-> Post already removed") + return + post.mod.remove(mod_note=reason) + if message is not None: + formatted_message = REMOVAL_MESSAGE_TEMPLATE.format(message=message) + post.mod.send_removal_message(formatted_message) + + +def is_removed(item): + return item.removed or item.banned_by is not None + + +########################################## +# OC fanart frequency verification block # +########################################## + + +def check_fanart_frequency(post): + count = 0 + for submission in post.author.submissions.new(): + if submission.subreddit.display_name == config_loader.REDDIT["subreddit"] and is_removed(submission): + continue + + created_at = datetime.fromtimestamp(submission.created_utc, tz=timezone.utc) + if datetime.now(timezone.utc) - created_at > timedelta(days=6, hours=23, minutes=45): + break + if is_fanart(submission): + count += 1 + if count > 2: + remove( + post, + f"Recent fanart (id: {submission.id})", + message="You may only submit two fanart posts in a 7-day period.", + ) + break + + logger.debug(f"Finished checking history of {post.author.name} for fanart frequency") + + +def is_fanart(post): + return post.subreddit.display_name == config_loader.REDDIT["subreddit"] and post.link_flair_text == "Fanart" + + +##################################### +# Clip frequency verification block # +##################################### + + +def check_clip_frequency(post): + count = 0 + for submission in post.author.submissions.new(): + if submission.subreddit.display_name == config_loader.REDDIT["subreddit"] and is_removed(submission): + continue + + created_at = datetime.fromtimestamp(submission.created_utc, tz=timezone.utc) + if datetime.now(timezone.utc) - created_at > timedelta(days=29, hours=23, minutes=45): + break + if is_clip(submission): + count += 1 + if count > 2: + remove(post, "Too many clips submitted", message="You may only submit two clips every 30 days.") + break + + logger.debug(f"Finished checking history of {post.author.name} for clip frequency") + + +def is_clip(post): + return post.subreddit.display_name == config_loader.REDDIT["subreddit"] and post.link_flair_text == "Clip" + + +##################################### +# Video Edit frequency verification block # +##################################### + + +def check_video_edit_frequency(post): + count = 0 + for submission in post.author.submissions.new(): + if submission.subreddit.display_name == config_loader.REDDIT["subreddit"] and is_removed(submission): + continue + + created_at = datetime.fromtimestamp(submission.created_utc, tz=timezone.utc) + if datetime.now(timezone.utc) - created_at > timedelta(days=29, hours=23, minutes=45): + break + if is_video_edit(submission): + count += 1 + if count > 2: + remove(post, "Too many clips submitted", message="You may only submit two video edits every 30 days.") + break + + logger.debug(f"Finished checking history of {post.author.name} for video edit frequency") + + +def is_video_edit(post): + return post.subreddit.display_name == config_loader.REDDIT["subreddit"] and post.link_flair_text == "Video Edit" + + +##################################### +# Video frequency verification block # +##################################### + + +def check_video_frequency(post): + count = 0 + for submission in post.author.submissions.new(): + if submission.subreddit.display_name == config_loader.REDDIT["subreddit"] and is_removed(submission): + continue + + created_at = datetime.fromtimestamp(submission.created_utc, tz=timezone.utc) + if datetime.now(timezone.utc) - created_at > timedelta(days=6, hours=23, minutes=45): + break + if is_video(submission): + count += 1 + if count > 2: + remove(post, "Too many videos submitted", message="You can only submit 2 videos at most every 7 days.") + break + + logger.debug(f"Finished checking history of {post.author.name} for video frequency") + + +def is_video(post): + return post.subreddit.display_name == config_loader.REDDIT["subreddit"] and post.link_flair_text == "Video" + + +##################################### + + +def monitor_stream(): + """ + Monitor the subreddit for new posts and parse them when they come in. Will restart upon encountering an error. + """ + + while True: + try: + logger.info("Connecting to Reddit...") + reddit = reddit_utils.get_reddit_instance(config_loader.REDDIT["auth"]) + subreddit = reddit.subreddit(config_loader.REDDIT["subreddit"]) + logger.info("Starting submission processing...") + main(subreddit) + except Exception: + delay_time = 30 + logger.exception(f"Encountered an unexpected error, restarting in {delay_time} seconds...") + time.sleep(delay_time) + + +if __name__ == "__main__": + DEBUG = False + monitor_stream() From 3432023f74a436ac783e6c8c1150bc38da6bae4d Mon Sep 17 00:00:00 2001 From: Durinthal Date: Sun, 1 Oct 2023 00:25:58 -0400 Subject: [PATCH 19/20] Handle deleted accounts for frontpage script --- scripts/frontpage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/frontpage.py b/scripts/frontpage.py index 321791d..ab2c0ed 100644 --- a/scripts/frontpage.py +++ b/scripts/frontpage.py @@ -33,7 +33,7 @@ def _format_line(submission, position, rank_change, total_hours): line += " {:>25}".format(f"[{submission.link_flair_text}]({submission.id})") - line += f" <{submission.author.name}>" + line += f" <{submission.author.name}>" if submission.author is not None else " <[deleted]>" line += f" <{reddit_utils.slug(submission)}>" From 3153802f8f7c3933a58ff589d415abb4f77113fd Mon Sep 17 00:00:00 2001 From: Durinthal Date: Thu, 26 Oct 2023 21:56:56 -0400 Subject: [PATCH 20/20] Remove BotDefense stat and fix mislabeled uniqies in monthly report --- scripts/reports.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/reports.py b/scripts/reports.py index 6bcb52c..35cba1b 100644 --- a/scripts/reports.py +++ b/scripts/reports.py @@ -104,9 +104,9 @@ def _report_monthly(report_args: argparse.Namespace): details="permanent", exclude_mod_accounts_list=mod_constants.ADMINS, ) - banned_users_bots = mod_action_service.count_mod_actions( - "banuser", start_date, end_date, distinct=True, mod_accounts_list=mod_constants.BOTS - ) + # banned_users_bots = mod_action_service.count_mod_actions( + # "banuser", start_date, end_date, distinct=True, mod_accounts_list=mod_constants.BOTS + # ) unbanned_users = mod_action_service.count_mod_actions( "unbanuser", start_date, end_date, distinct=True, exclude_mod_accounts_list=_bots_and_admins ) @@ -142,7 +142,7 @@ def _report_monthly(report_args: argparse.Namespace): meta_message = f"""Monthly Report – {start_date.strftime("%B %Y")}: ``` -- Total traffic: {total_views} pageviews, {unique_views} unique pageviews +- Total traffic: {total_views} pageviews, {unique_views} unique visitors - Total posts: {total_posts}, {total_post_authors} unique authors - Total comments: {total_comments}, {total_comment_authors} unique authors (excluding mod bots) - Removed posts: {removed_posts_humans} by moderators, {removed_posts_bots} by bots, {removed_posts_total} distinct @@ -150,7 +150,7 @@ def _report_monthly(report_args: argparse.Namespace): - Approved posts: {approved_posts} - Approved comments: {approved_comments} - Distinguished comments: {distinguished_comments} -- Users banned: {banned_users} ({permabanned_users} permanent, {banned_users_bots} by BotDefense) +- Users banned: {banned_users} ({permabanned_users} permanent) - Users unbanned: {actual_unbanned} - Admin/Anti-Evil Operations: removed posts: {admin_removed_posts}, removed comments: {admin_removed_comments}.```""" # noqa: E501