use osrs-package

.vscode/settings.json

@@ -1,11 +1,8 @@
 {
-    "python.testing.pytestArgs": [
-        "src"
-    ],
     "python.testing.unittestEnabled": false,
     "python.testing.pytestEnabled": true,
     "[python]": {
-        "editor.defaultFormatter": "ms-python.black-formatter",
+        "editor.defaultFormatter": "charliermarsh.ruff",
         "editor.formatOnSave": true,
         "editor.codeActionsOnSave": {
             "source.organizeImports": "explicit"

Dockerfile

@@ -1,5 +1,5 @@
 # Base stage
-FROM python:3.11-slim as base
+FROM python:3.11-slim AS base
 
 # Keeps Python from generating .pyc files in the container
 ENV PYTHONDONTWRITEBYTECODE=1
@@ -17,18 +17,16 @@ RUN pip install --no-cache-dir -r requirements.txt
 # copy the scripts to the folder
 COPY ./src /project/src
 
-# Production stage for highscore
-FROM base as production-highscore
+# Non-root user setup stage
+FROM base AS user-setup
 # Creates a non-root user with an explicit UID and adds permission to access the /project folder
 RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /project
 USER appuser
 
+# Production stage for highscore
+FROM user-setup AS production-highscore
 CMD ["python", "src/main_highscore.py"]
 
 # Production stage for runemetrics
-FROM base as production-runemetrics
-# Creates a non-root user with an explicit UID and adds permission to access the /project folder
-RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /project
-USER appuser
-
+FROM user-setup AS production-runemetrics
 CMD ["python", "src/main_runemetrics.py"]

Makefile

@@ -0,0 +1,33 @@
+clean-pyc: ## Clean Python cache files
+	find . \( -name '*.pyc' -o -name '*.pyo' -o -name '*~' -o -name '__pycache__' -o -name '.pytest_cache' \) -exec rm -rf {} +
+
+clean-test: ## Cleanup pytest leftovers
+	rm -f .coverage
+	rm -fr htmlcov/
+	rm -fr test_results/
+	rm -f *report.html log.html test-results.html output.xml
+
+docker-restart: ## Restart containers
+	docker compose down
+	docker compose up --build -d
+
+docker-test: docker-restart ## Restart containers & run tests
+	pytest
+
+docker-test-verbose: docker-restart ## Restart containers & run tests with verbosity
+	pytest -s
+
+pre-commit-setup: ## Install pre-commit
+	python3 -m pip install pre-commit
+	pre-commit --version
+
+requirements: ## Install all requirements
+	python3 -m pip install -r requirements.txt
+	python3 -m pip install -r requirements-test.txt
+	python3 -m pip install ruff
+
+setup: requirements pre-commit-setup## Set up all requirements
+
+docs: ## Open your browser to the web apps testing docs
+	@echo "Opening documentation..."
+	xdg-open http://localhost:5000/docs || open http://localhost:5000/docs

docker-compose-kafka.yml

@@ -1,4 +1,3 @@
-version: '3'
 services:
   kafka:
     container_name: kafka

docker-compose.yml

@@ -1,4 +1,3 @@
-version: '3'
 services:
   kafka:
     container_name: kafka
@@ -57,16 +56,20 @@ services:
     depends_on:
       kafka:
         condition: service_healthy
-
+        
   scraper_highscore:
-    image: bot_detector/scraper_highscore
-    container_name: scraper_highscore
+    image: bd/scraper_hiscore
+    container_name: scraper_hiscore
     build:
       context: .
       dockerfile: Dockerfile
       target: base
-    command: python src/main_highscore.py
+    command: >
+      sh -c "pip install watchdog && watchmedo auto-restart --patterns='*.py' --recursive -- python src/main.py --run-hs-worker"
     restart: on-failure
+    ports:
+      # host:container
+      - 8000:8000
     networks:
       - botdetector-network
     env_file:
@@ -76,16 +79,20 @@ services:
     depends_on:
       kafka:
         condition: service_healthy
-
+  
   scraper_runemetrics:
-    image: bot_detector/scraper_runemetrics
+    image: bd/scraper_runemetrics
     container_name: scraper_runemetrics
     build:
       context: .
       dockerfile: Dockerfile
       target: base
-    command: python src/main_runemetrics.py
+    command: >
+      sh -c "pip install watchdog && watchmedo auto-restart --patterns='*.py' --recursive -- python src/main.py --run-rm-worker"
     restart: on-failure
+    ports:
+      # host:container
+      - 8001:8000
     networks:
       - botdetector-network
     env_file:
@@ -95,5 +102,6 @@ services:
     depends_on:
       kafka:
         condition: service_healthy
+
 networks:
   botdetector-network:

requirements-test.txt

@@ -0,0 +1,2 @@
+pytest==8.3.3
+pytest-asyncio==0.24.0 

requirements.txt

@@ -1,30 +1,45 @@
-aiohttp==3.9.0
-aiokafka==0.8.1
-aiosignal==1.3.1
-async-timeout==4.0.2
-attrs==22.1.0
-black==22.12.0
-certifi==2023.7.22
-cffi==1.15.1
-chardet==5.1.0
-charset-normalizer==2.1.1
-click==8.1.3
-colorama==0.4.6
-frozenlist==1.3.3
-idna==3.4
-kafka-python==2.0.2
-multidict==6.0.3
-mypy-extensions==0.4.3
-packaging==23.1
-pathspec==0.10.3
-platformdirs==2.6.0
-pycares==4.3.0
-pycparser==2.21
-pydantic==1.10.8
-python-dotenv==0.21.0
-requests==2.31.0
-rfc3339==6.2
-tomli==2.0.1
-typing_extensions==4.4.0
-urllib3==1.26.13
-yarl==1.8.2
+aiohappyeyeballs==2.4.3
+aiohttp==3.10.10
+aiokafka==0.12.0
+aiosignal==1.3.1
+annotated-types==0.7.0
+async-timeout==4.0.3
+attrs==24.2.0
+black==24.10.0
+certifi==2024.8.30
+cffi==1.17.1
+cfgv==3.4.0
+chardet==5.2.0
+charset-normalizer==3.4.0
+click==8.1.7
+colorama==0.4.6
+distlib==0.3.9
+filelock==3.16.1
+frozenlist==1.5.0
+identify==2.6.1
+idna==3.10
+kafka-python==2.0.2
+multidict==6.1.0
+mypy-extensions==1.0.0
+nodeenv==1.9.1
+osrs==0.0.13
+packaging==24.1
+pathspec==0.12.1
+platformdirs==4.3.6
+pre_commit==4.0.1
+prometheus_client==0.21.0
+propcache==0.2.0
+pycares==4.4.0
+pycparser==2.22
+pydantic==2.9.2
+pydantic-settings==2.6.0
+pydantic_core==2.23.4
+python-dotenv==1.0.1
+PyYAML==6.0.2
+requests==2.32.3
+rfc3339==6.2
+tomli==2.0.2
+typing_extensions==4.12.2
+urllib3==2.2.3
+virtualenv==20.27.1
+yarl==1.17.0

src/config/__init__.py

@@ -0,0 +1,3 @@
+from config import logging
+
+__all__ = ["logging"]

src/config/config.py

@@ -1,8 +1,6 @@
 import os
 
-from pydantic import BaseSettings
-
-from config import logging
+from pydantic_settings import BaseSettings
 
 
 class AppConfig(BaseSettings):

src/config/logging.py

@@ -18,9 +18,21 @@
     # file_handler # this is good for debugging
 ]
 
+
+# | Level                | Numeric Value | What it Means / When to Use It                                                                 |
+# |----------------------|---------------|--------------------------------------------------------------------------------------------------|
+# | `logging.NOTSET`     | 0             | When set on a logger, indicates that ancestor loggers are to be consulted to determine the effective level. If that still resolves to NOTSET, then all events are logged. When set on a handler, all events are handled. |
+# | `logging.DEBUG`      | 10            | Detailed information, typically only of interest to a developer trying to diagnose a problem.  |
+# | `logging.INFO`       | 20            | Confirmation that things are working as expected.                                               |
+# | `logging.WARNING`    | 30            | An indication that something unexpected happened, or that a problem might occur in the near future (e.g., ‘disk space low’). The software is still working as expected. |
+# | `logging.ERROR`      | 40            | Due to a more serious problem, the software has not been able to perform some function.         |
+# | `logging.CRITICAL`   | 50            | A serious error, indicating that the program itself may be unable to continue running.          |
+
 logging.basicConfig(level=logging.DEBUG, handlers=handlers)
 
 logging.getLogger("urllib3").setLevel(logging.INFO)
 # logging.getLogger("modules.scraper").setLevel(logging.WARNING)
 logging.getLogger("modules.api").setLevel(logging.WARNING)
 logging.getLogger("aiokafka").setLevel(logging.INFO)
+logging.getLogger("aiokafka").setLevel(logging.INFO)
+logging.getLogger("osrs.async_api.osrs.hiscores").setLevel(logging.INFO)