From cb0acb74635bb51912c3384d9be802c6b70ad580 Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Sun, 17 Nov 2024 16:11:27 +0100 Subject: [PATCH 1/2] mono repo style --- src/config/__init__.py | 2 +- src/config/config.py | 2 +- src/config/logging.py | 50 +++++++++++------------- src/jobs/tipoff/main.py | 4 -- src/main.py | 86 ++++++++++++++++++----------------------- 5 files changed, 63 insertions(+), 81 deletions(-) diff --git a/src/config/__init__.py b/src/config/__init__.py index 077d213..d4c5213 100644 --- a/src/config/__init__.py +++ b/src/config/__init__.py @@ -1,3 +1,3 @@ from . import logging -logging.handlers \ No newline at end of file +__all__ = ["logging"] diff --git a/src/config/config.py b/src/config/config.py index d9553d5..bf48dc2 100644 --- a/src/config/config.py +++ b/src/config/config.py @@ -1,7 +1,7 @@ import os import dotenv -from pydantic import BaseSettings +from pydantic_settings import BaseSettings dotenv.load_dotenv() diff --git a/src/config/logging.py b/src/config/logging.py index 419397e..86ede60 100644 --- a/src/config/logging.py +++ b/src/config/logging.py @@ -2,36 +2,32 @@ import logging import sys -# setup logging -file_handler = logging.FileHandler(filename="./error.log", mode="a") -stream_handler = logging.StreamHandler(sys.stdout) -# # log formatting -formatter = logging.Formatter( - json.dumps( - { - "ts": "%(asctime)s", - "name": "%(name)s", - "function": "%(funcName)s", - "level": "%(levelname)s", - "msg": "%(message)s", + +# Configure JSON logging +class JsonFormatter(logging.Formatter): + def format(self, record): + log_record = { + "ts": self.formatTime(record, self.datefmt), + "lvl": record.levelname, + "module": record.module, + "funcName": record.funcName, + "lineNo": record.lineno, + "msg": record.getMessage(), } - ) -) + if record.exc_info: + log_record["exception"] = self.formatException(record.exc_info) + return json.dumps(log_record) -file_handler.setFormatter(formatter) -stream_handler.setFormatter(formatter) +class IgnoreSpecificWarnings(logging.Filter): + def filter(self, record): + # Return False to filter out messages containing "Unknown table" + return "Unknown table" not in record.getMessage() -handlers = [ - # file_handler, - stream_handler -] -logging.basicConfig(level=logging.DEBUG, handlers=handlers) +# Set up the logger +handler = logging.StreamHandler(sys.stdout) +handler.setFormatter(JsonFormatter()) -logging.getLogger("mysql.connector.connection").setLevel(logging.WARNING) -logging.getLogger("urllib3.connectionpool").setLevel(logging.WARNING) -logging.getLogger("requests_oauthlib").setLevel(logging.WARNING) -logging.getLogger("oauthlib").setLevel(logging.WARNING) -logging.getLogger("discord.webhook.sync").setLevel(logging.WARNING) -logging.getLogger("aiokafka").setLevel(logging.WARNING) +logging.basicConfig(level=logging.INFO, handlers=[handler]) +logging.getLogger("asyncmy").addFilter(IgnoreSpecificWarnings()) diff --git a/src/jobs/tipoff/main.py b/src/jobs/tipoff/main.py index 4cb9950..671cdba 100644 --- a/src/jobs/tipoff/main.py +++ b/src/jobs/tipoff/main.py @@ -59,7 +59,3 @@ def tipoff_bots() -> None: call_webhook(path=PATH_TO_CSV_FILE, url=config.PATRON_WEBHOOK) return - - -if __name__ == "__main__": - tipoff_bots() diff --git a/src/main.py b/src/main.py index 261d086..73ae090 100644 --- a/src/main.py +++ b/src/main.py @@ -1,48 +1,38 @@ -import asyncio -import logging - -from apscheduler.executors.pool import ProcessPoolExecutor -from apscheduler.schedulers.background import BackgroundScheduler - -from jobs.banbroadcaster.main import broadcast_bans -from jobs.kafka.players.main import get_players_to_scrape -from jobs.tipoff.main import tipoff_bots - -logger = logging.getLogger(__name__) - -executors = {"default": ProcessPoolExecutor()} - -scheduler = BackgroundScheduler(daemon=False, executors=executors) - - -async def main(): - logger.debug("Starting Main") - scheduler.start() - - # Broadcast bans to #bot-graveyard on Discord, send total bans tweet, and send bans breakdowns tweets - scheduler.add_job( - broadcast_bans, - "cron", - hour=20, - minute=5, - misfire_grace_time=60, - ) - - # Send next tipoff batch to Jagex - scheduler.add_job( - tipoff_bots, - "cron", - hour=9, - minute=45, - misfire_grace_time=60, - ) - - get_players_to_scrape() - # broadcast_bans() - # tipoff_bots() - while True: - await asyncio.sleep(60) - - -if __name__ == "__main__": - asyncio.run(main()) +import argparse + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--run-broadcast", + action="store_true", + ) + parser.add_argument( + "--run-tipoff", + action="store_true", + ) + parser.add_argument( + "--run-kafka", + action="store_true", + ) + + args = parser.parse_args() + + if args.run_broadcast: + from jobs.banbroadcaster.main import broadcast_bans + + broadcast_bans() + elif args.run_tipoff: + from jobs.tipoff.main import tipoff_bots + + tipoff_bots() + elif args.run_kafka: + from jobs.kafka.players.main import get_players_to_scrape + + get_players_to_scrape() + else: + parser.print_help() + + +if __name__ == "__main__": + main() From bb91e84f241c5d01643451895e5eee5ad42169e5 Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Sun, 17 Nov 2024 21:36:21 +0100 Subject: [PATCH 2/2] cleanup --- .env-example | 1 + .pre-commit-config.yaml | 10 ---- deployment.yaml | 24 -------- src/jobs/banbroadcaster/functions.py | 86 ++++++++++++++-------------- src/jobs/kafka/players/main.py | 2 +- 5 files changed, 46 insertions(+), 77 deletions(-) delete mode 100644 .pre-commit-config.yaml delete mode 100644 deployment.yaml diff --git a/.env-example b/.env-example index f83e37b..dd93b85 100644 --- a/.env-example +++ b/.env-example @@ -8,6 +8,7 @@ JMOD_TAG = SERVER_LOGIN = SERVER_PASSWORD = SERVER_ADDRESS = +SERVER_PORT=3306 DATABASE = GRAVEYARD_WEBHOOK = diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index 06de33f..0000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,10 +0,0 @@ -repos: - - repo: https://github.com/psf/black - rev: 23.10.1 - hooks: - - id: black - - repo: https://github.com/pycqa/isort - rev: 5.12.0 - hooks: - - id: isort - args: ["--profile", "black"] diff --git a/deployment.yaml b/deployment.yaml deleted file mode 100644 index 27bf6c4..0000000 --- a/deployment.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: bd-automations -spec: - selector: - matchLabels: - app: bd-automations - replicas: 1 - template: - metadata: - labels: - app: bd-automations - spec: - containers: - - name: bd-automations - image: hub.osrsbotdetector.com/bot-detector/bd-automations:latest - imagePullPolicy: Always - resources: - limits: - cpu: "500m" - memory: "750Mi" - imagePullSecrets: - - name: regcred diff --git a/src/jobs/banbroadcaster/functions.py b/src/jobs/banbroadcaster/functions.py index 73cbe0a..887badf 100644 --- a/src/jobs/banbroadcaster/functions.py +++ b/src/jobs/banbroadcaster/functions.py @@ -2,11 +2,13 @@ import operator import time from collections import namedtuple -from datetime import datetime + +# from datetime import datetime from typing import List import mysql.connector -import tweepy + +# import tweepy from discord_webhook import DiscordWebhook from discord_webhook.webhook import DiscordEmbed @@ -16,11 +18,11 @@ logger = logging.getLogger(__name__) -AUTH = tweepy.OAuthHandler( - consumer_key=config.CONSUMER_KEY, consumer_secret=config.CONSUMER_SECRET -) -AUTH.set_access_token(key=config.ACCESS_TOKEN, secret=config.ACCESS_TOKEN_SECRET) -TWITTER_API = tweepy.API(AUTH, wait_on_rate_limit=True) +# AUTH = tweepy.OAuthHandler( +# consumer_key=config.CONSUMER_KEY, consumer_secret=config.CONSUMER_SECRET +# ) +# AUTH.set_access_token(key=config.ACCESS_TOKEN, secret=config.ACCESS_TOKEN_SECRET) +# TWITTER_API = tweepy.API(AUTH, wait_on_rate_limit=True) config_players = { @@ -175,57 +177,57 @@ def broadcast_names(names_list: List[str]): time.sleep(5) -def post_bans_tweet(num_bans: int): - logger.debug("Posting Ban Tweet") - msg = f"BANS ALERT - {datetime.now().strftime('%d-%m-%Y')}: {num_bans:,d} accounts our system has detected as bots have been banned in the past 24 hours." - TWITTER_API.update_status(msg) +# def post_bans_tweet(num_bans: int): +# logger.debug("Posting Ban Tweet") +# msg = f"BANS ALERT - {datetime.now().strftime('%d-%m-%Y')}: {num_bans:,d} accounts our system has detected as bots have been banned in the past 24 hours." +# TWITTER_API.update_status(msg) -def post_breakdown_tweets(predictions: List[str]): - logger.debug("Posting Breakdown Tweet") - tweets = generate_breakdown_tweets(predictions) +# def post_breakdown_tweets(predictions: List[str]): +# logger.debug("Posting Breakdown Tweet") +# tweets = generate_breakdown_tweets(predictions) - if len(tweets) == 0: - return +# if len(tweets) == 0: +# return - previous_status = None +# previous_status = None - for i, tweet in enumerate(tweets): - tweet += f"({i+1}/{len(tweets)})" +# for i, tweet in enumerate(tweets): +# tweet += f"({i+1}/{len(tweets)})" - if i == 0: - previous_status = TWITTER_API.update_status(tweet) - else: - previous_status = TWITTER_API.update_status( - tweet, in_reply_to_status_id=previous_status.id - ) +# if i == 0: +# previous_status = TWITTER_API.update_status(tweet) +# else: +# previous_status = TWITTER_API.update_status( +# tweet, in_reply_to_status_id=previous_status.id +# ) - time.sleep(3) +# time.sleep(3) - return +# return -def generate_breakdown_tweets(predictions: List[str]): - logger.info("Generating Breakdown Tweet") - predictions_groupings = group_predictions(predictions) +# def generate_breakdown_tweets(predictions: List[str]): +# logger.info("Generating Breakdown Tweet") +# predictions_groupings = group_predictions(predictions) - tweets = [] +# tweets = [] - current_tweet = "Bans by Category:\n" +# current_tweet = "Bans by Category:\n" - for pred, count in predictions_groupings.items(): - pred_string = f"{pred}: {count}\n" +# for pred, count in predictions_groupings.items(): +# pred_string = f"{pred}: {count}\n" - if (len(current_tweet) + len(pred_string)) >= 240: - tweets.append(current_tweet) - current_tweet = pred_string - else: - current_tweet += pred_string +# if (len(current_tweet) + len(pred_string)) >= 240: +# tweets.append(current_tweet) +# current_tweet = pred_string +# else: +# current_tweet += pred_string - # Grab the leftovers! - tweets.append(current_tweet) +# # Grab the leftovers! +# tweets.append(current_tweet) - return tweets +# return tweets def group_predictions(predictions: List[str]): diff --git a/src/jobs/kafka/players/main.py b/src/jobs/kafka/players/main.py index a8c0194..8f1a086 100644 --- a/src/jobs/kafka/players/main.py +++ b/src/jobs/kafka/players/main.py @@ -243,4 +243,4 @@ async def main(): def get_players_to_scrape(): - asyncio.ensure_future(main()) + asyncio.run(main())