improved logging

.pre-commit-config.yaml

@@ -1,9 +1,12 @@
 repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.3.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.7.4
     hooks:
-      - id: check-yaml
-  - repo: https://github.com/psf/black
-    rev: 22.10.0
-    hooks:
-      - id: black
+      # Run the linter.
+      - id: ruff
+        types_or: [python, pyi]
+        args: [--fix]
+      # Run the formatter.
+      - id: ruff-format
+        types_or: [python, pyi]

.python-version

@@ -0,0 +1 @@
+3.12

pyproject.toml

@@ -0,0 +1,24 @@
+[project]
+name = "core-api"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "aiohttp==3.9.5",
+    "asyncmy==0.2.8",
+    "fastapi==0.100.1",
+    "pandas>=2.0.3",
+    "python-dotenv==1.0.0",
+    "sqlalchemy==2.0.19",
+    "starlette-prometheus==0.9.0",
+    "uvicorn==0.23.1",
+]
+
+[dependency-groups]
+dev = [
+    "httpx>=0.28.0",
+    "pytest-asyncio>=0.24.0",
+    "pytest>=8.3.3",
+    "ruff>=0.8.1",
+]

src/api/legacy/__init__.py

@@ -1,8 +1,8 @@
 from fastapi import APIRouter
 
-from src.api.legacy import legacy, legacy_debug
+from src.api.legacy import legacy
 
 router = APIRouter()
 
 router.include_router(legacy.router)
-router.include_router(legacy_debug.router)
+# router.include_router(legacy_debug.router)

src/api/legacy/legacy.py

@@ -826,80 +826,6 @@ async def receive_plugin_feedback(feedback: Feedback, version: str = None):
 
     return {"OK": "OK"}
 
-
-@router.get("/site/highscores/{token}/{ofInterest}", tags=["Legacy"])
-@router.get("/site/highscores/{token}/{ofInterest}/{row_count}/{page}", tags=["Legacy"])
-async def get_highscores(
-    token: str,
-    request: Request,
-    ofInterest: int = None,
-    row_count: Optional[int] = 100_000,
-    page: Optional[int] = 1,
-):
-    await verify_token(
-        token,
-        verification="request_highscores",
-        route=logging_helpers.build_route_log_string(request, [token]),
-    )
-
-    if ofInterest is None:
-        sql = """
-            SELECT
-                hdl.*,
-                pl.name
-            FROM playerHiscoreDataLatest hdl
-            inner join Players pl on(hdl.Player_id=pl.id)
-        """
-    else:
-        sql = """
-            SELECT
-                htl.*,
-                poi.name
-            FROM playerHiscoreDataLatest htl
-            INNER JOIN playersOfInterest poi ON (htl.Player_id = poi.id)
-        """
-
-    data = await execute_sql(sql, row_count=row_count, page=page)
-    return data.rows2dict() if data is not None else {}
-
-
-@router.get("site/players/{token}/{ofInterest}/{row_count}/{page}", tags=["Legacy"])
-async def get_players(
-    token: str,
-    request: Request,
-    ofInterest: int = None,
-    row_count: int = 100_000,
-    page: int = 1,
-):
-    await verify_token(
-        token,
-        verification="request_highscores",
-        route=logging_helpers.build_route_log_string(request, [token]),
-    )
-
-    # get data
-    if ofInterest is None:
-        sql = "select * from Players"
-    else:
-        sql = "select * from playersOfInterest"
-
-    data = await execute_sql(sql, row_count=row_count, page=page)
-    return data.rows2dict() if data is not None else {}
-
-
-@router.get("/site/labels/{token}", tags=["Legacy"])
-async def get_labels(token, request: Request):
-    await verify_token(
-        token,
-        verification="request_highscores",
-        route=logging_helpers.build_route_log_string(request, [token]),
-    )
-
-    sql = "select * from Labels"
-    data = await execute_sql(sql)
-    return data.rows2dict() if data is not None else {}
-
-
 @router.post("/site/verify/{token}", tags=["Legacy"])
 async def verify_bot(token: str, bots: bots, request: Request):
     await verify_token(

src/core/__init__.py

@@ -0,0 +1,3 @@
+from . import config, logging
+
+__all__ = ["logging", "config"]

src/core/config.py

@@ -14,50 +14,4 @@
 discord_sql_uri = os.environ.get("discord_sql_uri")
 token = os.environ.get("token")
 kafka_url = os.environ.get("kafka_url", "127.0.0.1:9094")
-env = os.environ.get("env", "DEV")
-
-# setup logging
-file_handler = logging.FileHandler(filename="./error.log", mode="a")
-stream_handler = logging.StreamHandler(sys.stdout)
-# # log formatting
-formatter = logging.Formatter(
-    json.dumps(
-        {
-            "ts": "%(asctime)s",
-            "name": "%(name)s",
-            "function": "%(funcName)s",
-            "level": "%(levelname)s",
-            "msg": json.dumps("%(message)s"),
-        }
-    )
-)
-
-
-file_handler.setFormatter(formatter)
-stream_handler.setFormatter(formatter)
-
-handlers = [
-    # file_handler,
-    stream_handler
-]
-
-logging.basicConfig(level=logging.DEBUG, handlers=handlers)
-
-# set imported loggers to warning
-logging.getLogger("requests").setLevel(logging.DEBUG)
-logging.getLogger("urllib3").setLevel(logging.DEBUG)
-logging.getLogger("uvicorn").setLevel(logging.DEBUG)
-logging.getLogger("apscheduler").setLevel(logging.WARNING)
-logging.getLogger("aiomysql").setLevel(logging.ERROR)
-logging.getLogger("asyncmy").setLevel(logging.ERROR)
-logging.getLogger("aiokafka").setLevel(logging.WARNING)
-
-if env != "DEV":
-    uvicorn_error = logging.getLogger("uvicorn.error")
-    uvicorn_error.disabled = True
-    uvicorn_access = logging.getLogger("uvicorn.access")
-    uvicorn_access.disabled = True
-
-# https://github.com/aio-libs/aiomysql/issues/103
-# https://github.com/coleifer/peewee/issues/2229
-warnings.filterwarnings("ignore", ".*Duplicate entry.*")
+env = os.environ.get("env", "DEV")

src/core/logging.py

@@ -0,0 +1,44 @@
+import json
+import logging
+
+
+# Configure JSON logging
+class JsonFormatter(logging.Formatter):
+    def format(self, record):
+        log_record = {
+            "ts": self.formatTime(record, self.datefmt),
+            "lvl": record.levelname,
+            "name": record.name,
+            # "module": record.module,
+            "func": record.funcName,
+            "line": record.lineno,
+            "msg": record.getMessage(),
+        }
+        if record.exc_info:
+            log_record["exception"] = self.formatException(record.exc_info)
+        return json.dumps(log_record)
+
+
+class IgnoreSQLWarnings(logging.Filter):
+    def filter(self, record):
+        ignore_messages = ["Unknown table", "Duplicate entry"]
+        # Check if any of the ignore messages are in the log record message
+        if any(msg in record.getMessage() for msg in ignore_messages):
+            return False  # Don't log
+        return True  # Log
+
+
+# Set up the logger
+handler = logging.StreamHandler()
+handler.setFormatter(JsonFormatter())
+
+logging.basicConfig(level=logging.INFO, handlers=[handler])
+
+# set imported loggers to warning
+logging.getLogger("requests").setLevel(logging.DEBUG)
+logging.getLogger("urllib3").setLevel(logging.DEBUG)
+logging.getLogger("uvicorn").setLevel(logging.DEBUG)
+logging.getLogger("apscheduler").setLevel(logging.WARNING)
+logging.getLogger("aiomysql").setLevel(logging.ERROR)
+logging.getLogger("asyncmy").setLevel(logging.ERROR)
+logging.getLogger("aiokafka").setLevel(logging.WARNING)