Merge pull request #47 from Bot-detector/mono-repo

mono repo style

.env-example

@@ -8,6 +8,7 @@ JMOD_TAG =
 SERVER_LOGIN = 
 SERVER_PASSWORD = 
 SERVER_ADDRESS = 
+SERVER_PORT=3306
 DATABASE = 
 
 GRAVEYARD_WEBHOOK = 

src/config/__init__.py

@@ -1,3 +1,3 @@
 from . import logging
 
-logging.handlers
+__all__ = ["logging"]

src/config/config.py

@@ -1,7 +1,7 @@
 import os
 
 import dotenv
-from pydantic import BaseSettings
+from pydantic_settings import BaseSettings
 
 dotenv.load_dotenv()
 

src/config/logging.py

@@ -2,36 +2,32 @@
 import logging
 import sys
 
-# setup logging
-file_handler = logging.FileHandler(filename="./error.log", mode="a")
-stream_handler = logging.StreamHandler(sys.stdout)
-# # log formatting
-formatter = logging.Formatter(
-    json.dumps(
-        {
-            "ts": "%(asctime)s",
-            "name": "%(name)s",
-            "function": "%(funcName)s",
-            "level": "%(levelname)s",
-            "msg": "%(message)s",
+
+# Configure JSON logging
+class JsonFormatter(logging.Formatter):
+    def format(self, record):
+        log_record = {
+            "ts": self.formatTime(record, self.datefmt),
+            "lvl": record.levelname,
+            "module": record.module,
+            "funcName": record.funcName,
+            "lineNo": record.lineno,
+            "msg": record.getMessage(),
         }
-    )
-)
+        if record.exc_info:
+            log_record["exception"] = self.formatException(record.exc_info)
+        return json.dumps(log_record)
 
 
-file_handler.setFormatter(formatter)
-stream_handler.setFormatter(formatter)
+class IgnoreSpecificWarnings(logging.Filter):
+    def filter(self, record):
+        # Return False to filter out messages containing "Unknown table"
+        return "Unknown table" not in record.getMessage()
 
-handlers = [
-    # file_handler,
-    stream_handler
-]
 
-logging.basicConfig(level=logging.DEBUG, handlers=handlers)
+# Set up the logger
+handler = logging.StreamHandler(sys.stdout)
+handler.setFormatter(JsonFormatter())
 
-logging.getLogger("mysql.connector.connection").setLevel(logging.WARNING)
-logging.getLogger("urllib3.connectionpool").setLevel(logging.WARNING)
-logging.getLogger("requests_oauthlib").setLevel(logging.WARNING)
-logging.getLogger("oauthlib").setLevel(logging.WARNING)
-logging.getLogger("discord.webhook.sync").setLevel(logging.WARNING)
-logging.getLogger("aiokafka").setLevel(logging.WARNING)
+logging.basicConfig(level=logging.INFO, handlers=[handler])
+logging.getLogger("asyncmy").addFilter(IgnoreSpecificWarnings())

src/jobs/banbroadcaster/functions.py

@@ -2,11 +2,13 @@
 import operator
 import time
 from collections import namedtuple
-from datetime import datetime
+
+# from datetime import datetime
 from typing import List
 
 import mysql.connector
-import tweepy
+
+# import tweepy
 from discord_webhook import DiscordWebhook
 from discord_webhook.webhook import DiscordEmbed
 
@@ -16,11 +18,11 @@
 
 logger = logging.getLogger(__name__)
 
-AUTH = tweepy.OAuthHandler(
-    consumer_key=config.CONSUMER_KEY, consumer_secret=config.CONSUMER_SECRET
-)
-AUTH.set_access_token(key=config.ACCESS_TOKEN, secret=config.ACCESS_TOKEN_SECRET)
-TWITTER_API = tweepy.API(AUTH, wait_on_rate_limit=True)
+# AUTH = tweepy.OAuthHandler(
+#     consumer_key=config.CONSUMER_KEY, consumer_secret=config.CONSUMER_SECRET
+# )
+# AUTH.set_access_token(key=config.ACCESS_TOKEN, secret=config.ACCESS_TOKEN_SECRET)
+# TWITTER_API = tweepy.API(AUTH, wait_on_rate_limit=True)
 
 
 config_players = {
@@ -175,57 +177,57 @@ def broadcast_names(names_list: List[str]):
         time.sleep(5)
 
 
-def post_bans_tweet(num_bans: int):
-    logger.debug("Posting Ban Tweet")
-    msg = f"BANS ALERT - {datetime.now().strftime('%d-%m-%Y')}: {num_bans:,d} accounts our system has detected as bots have been banned in the past 24 hours."
-    TWITTER_API.update_status(msg)
+# def post_bans_tweet(num_bans: int):
+#     logger.debug("Posting Ban Tweet")
+#     msg = f"BANS ALERT - {datetime.now().strftime('%d-%m-%Y')}: {num_bans:,d} accounts our system has detected as bots have been banned in the past 24 hours."
+#     TWITTER_API.update_status(msg)
 
 
-def post_breakdown_tweets(predictions: List[str]):
-    logger.debug("Posting Breakdown Tweet")
-    tweets = generate_breakdown_tweets(predictions)
+# def post_breakdown_tweets(predictions: List[str]):
+#     logger.debug("Posting Breakdown Tweet")
+#     tweets = generate_breakdown_tweets(predictions)
 
-    if len(tweets) == 0:
-        return
+#     if len(tweets) == 0:
+#         return
 
-    previous_status = None
+#     previous_status = None
 
-    for i, tweet in enumerate(tweets):
-        tweet += f"({i+1}/{len(tweets)})"
+#     for i, tweet in enumerate(tweets):
+#         tweet += f"({i+1}/{len(tweets)})"
 
-        if i == 0:
-            previous_status = TWITTER_API.update_status(tweet)
-        else:
-            previous_status = TWITTER_API.update_status(
-                tweet, in_reply_to_status_id=previous_status.id
-            )
+#         if i == 0:
+#             previous_status = TWITTER_API.update_status(tweet)
+#         else:
+#             previous_status = TWITTER_API.update_status(
+#                 tweet, in_reply_to_status_id=previous_status.id
+#             )
 
-        time.sleep(3)
+#         time.sleep(3)
 
-    return
+#     return
 
 
-def generate_breakdown_tweets(predictions: List[str]):
-    logger.info("Generating Breakdown Tweet")
-    predictions_groupings = group_predictions(predictions)
+# def generate_breakdown_tweets(predictions: List[str]):
+#     logger.info("Generating Breakdown Tweet")
+#     predictions_groupings = group_predictions(predictions)
 
-    tweets = []
+#     tweets = []
 
-    current_tweet = "Bans by Category:\n"
+#     current_tweet = "Bans by Category:\n"
 
-    for pred, count in predictions_groupings.items():
-        pred_string = f"{pred}: {count}\n"
+#     for pred, count in predictions_groupings.items():
+#         pred_string = f"{pred}: {count}\n"
 
-        if (len(current_tweet) + len(pred_string)) >= 240:
-            tweets.append(current_tweet)
-            current_tweet = pred_string
-        else:
-            current_tweet += pred_string
+#         if (len(current_tweet) + len(pred_string)) >= 240:
+#             tweets.append(current_tweet)
+#             current_tweet = pred_string
+#         else:
+#             current_tweet += pred_string
 
-    # Grab the leftovers!
-    tweets.append(current_tweet)
+#     # Grab the leftovers!
+#     tweets.append(current_tweet)
 
-    return tweets
+#     return tweets
 
 
 def group_predictions(predictions: List[str]):

src/jobs/kafka/players/main.py

@@ -243,4 +243,4 @@ async def main():
 
 
 def get_players_to_scrape():
-    asyncio.ensure_future(main())
+    asyncio.run(main())

src/jobs/tipoff/main.py

@@ -59,7 +59,3 @@ def tipoff_bots() -> None:
 
     call_webhook(path=PATH_TO_CSV_FILE, url=config.PATRON_WEBHOOK)
     return
-
-
-if __name__ == "__main__":
-    tipoff_bots()

src/main.py

@@ -1,48 +1,38 @@
-import asyncio
-import logging
-
-from apscheduler.executors.pool import ProcessPoolExecutor
-from apscheduler.schedulers.background import BackgroundScheduler
-
-from jobs.banbroadcaster.main import broadcast_bans
-from jobs.kafka.players.main import get_players_to_scrape
-from jobs.tipoff.main import tipoff_bots
-
-logger = logging.getLogger(__name__)
-
-executors = {"default": ProcessPoolExecutor()}
-
-scheduler = BackgroundScheduler(daemon=False, executors=executors)
-
-
-async def main():
-    logger.debug("Starting Main")
-    scheduler.start()
-
-    # Broadcast bans to #bot-graveyard on Discord, send total bans tweet, and send bans breakdowns tweets
-    scheduler.add_job(
-        broadcast_bans,
-        "cron",
-        hour=20,
-        minute=5,
-        misfire_grace_time=60,
-    )
-
-    # Send next tipoff batch to Jagex
-    scheduler.add_job(
-        tipoff_bots,
-        "cron",
-        hour=9,
-        minute=45,
-        misfire_grace_time=60,
-    )
-
-    get_players_to_scrape()
-    # broadcast_bans()
-    # tipoff_bots()
-    while True:
-        await asyncio.sleep(60)
-
-
-if __name__ == "__main__":
-    asyncio.run(main())
+import argparse
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--run-broadcast",
+        action="store_true",
+    )
+    parser.add_argument(
+        "--run-tipoff",
+        action="store_true",
+    )
+    parser.add_argument(
+        "--run-kafka",
+        action="store_true",
+    )
+
+    args = parser.parse_args()
+
+    if args.run_broadcast:
+        from jobs.banbroadcaster.main import broadcast_bans
+
+        broadcast_bans()
+    elif args.run_tipoff:
+        from jobs.tipoff.main import tipoff_bots
+
+        tipoff_bots()
+    elif args.run_kafka:
+        from jobs.kafka.players.main import get_players_to_scrape
+
+        get_players_to_scrape()
+    else:
+        parser.print_help()
+
+
+if __name__ == "__main__":
+    main()