diff --git a/Makefile b/Makefile index 12b2fd82..8388d64a 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ builddeps: serverdeps: sudo apt-get update - sudo apt-get -y --allow-unauthenticated install nginx supervisor rabbitmq-server locales + sudo apt-get -y --allow-unauthenticated install nginx supervisor locales pip3 install -r config/requirements/production.pip --exists-action=s deps: diff --git a/accounts/tasks.py b/accounts/tasks.py index 7683f6bf..47cc070f 100644 --- a/accounts/tasks.py +++ b/accounts/tasks.py @@ -1,6 +1,5 @@ import logging -from celery import task from django.db import IntegrityError import games.models @@ -9,10 +8,12 @@ from accounts.models import User from common.util import slugify +from lutrisweb import celery_app + LOGGER = logging.getLogger() -@task +@celery_app.task def sync_steam_library(user_id): user = User.objects.get(pk=user_id) steamid = user.steamid @@ -50,6 +51,6 @@ def sync_steam_library(user_id): pass -@task +@celery_app.task def daily_mod_mail(): send_daily_mod_mail() diff --git a/config/requirements/base.pip b/config/requirements/base.pip index 0b71119d..8679140f 100644 --- a/config/requirements/base.pip +++ b/config/requirements/base.pip @@ -1,7 +1,7 @@ beautifulsoup4==4.6.0 PyYAML==3.12 ipython==6.4.0 -Celery==4.1.0 +Celery==4.4.2 cssselect==1.0.3 Django==2.0.5 django-appconf==1.0.2 diff --git a/docker-compose.yml b/docker-compose.yml index a4e274da..63add7af 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,9 @@ services: context: . args: - USE_SQLITE=${USE_SQLITE} + environment: + MOBY_API_KEY + restart: on-failure volumes: - .:/app - /app/media @@ -18,13 +21,20 @@ services: tty: true depends_on: - db + - rabbitmq db: image: postgres:11 ports: - 5432:5432 + restart: on-failure volumes: - postgresql:/var/lib/postgresql/data environment: POSTGRES_DB: 'lutris' POSTGRES_USER: 'lutris' POSTGRES_PASSWORD: 'admin' + rabbitmq: + image: rabbitmq + ports: + - 5672:5672 + restart: on-failure diff --git a/games/tasks.py b/games/tasks.py index a580af3d..9004b07c 100644 --- a/games/tasks.py +++ b/games/tasks.py @@ -1,21 +1,24 @@ """Celery tasks for account related jobs""" import logging -from celery import task + +import requests +from django.conf import settings from reversion.models import Version, Revision from django.contrib.contenttypes.models import ContentType from games import models +from lutrisweb import celery_app LOGGER = logging.getLogger(__name__) -@task +@celery_app.task def delete_unchanged_forks(): """Periodically delete forked installers that haven't received any changes""" for installer in models.Installer.objects.abandoned(): installer.delete() -@task +@celery_app.task def clear_orphan_versions(): """Deletes versions that are no longer associated with an installer""" content_type = ContentType.objects.get_for_model(models.Installer) @@ -26,7 +29,42 @@ def clear_orphan_versions(): version.delete() -@task +@celery_app.task def clear_orphan_revisions(): """Clear revisions that are no longer attached to any object""" Revision.objects.filter(version__isnull=True).delete() + + +@celery_app.task +def add_new_games(): + """Check MobyGames for newly added games every 3 hours and update Lutris DB""" + # + response = requests.get(url="https://api.mobygames.com/v1/games/recent", + params={ + 'api_key': settings.MOBY_API_KEY, + 'format': 'normal', + 'age': 1 + }) + if response.status_code == 200: + new_entries = 0 + data = response.json() + supported_platforms = models.Platform.objects.values_list('name', flat=True) + supported_genres = models.Genre.objects.values_list('name', flat=True) + for game in data.get('games', []): + game_platforms = [platform.get('platform_name', '') for platform in game.get('platforms', []) + if platform.get('platform_name', '') in supported_platforms] + game_genres = [genre.get('genre_name', '') for genre in game.get('genres', []) + if genre.get('genre_name', '') in supported_genres] + if game_platforms: + obj, created = models.Game.objects.get_or_create( + name=game.get('title', ''), + defaults={ + 'description': game.get('description', '') + } + ) + if created: + new_entries += 1 + obj.platforms.add(*models.Platform.objects.filter(name__in=game_platforms)) + obj.genres.add(*models.Genre.objects.filter(name__in=game_genres)) + else: + pass diff --git a/lutrisweb/__init__.py b/lutrisweb/__init__.py index 39c1a2eb..d14b5526 100644 --- a/lutrisweb/__init__.py +++ b/lutrisweb/__init__.py @@ -1,3 +1,7 @@ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals -from .celery import app # noqa +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery import app as celery_app # noqa + +__all__ = ('celery_app',) diff --git a/lutrisweb/celery.py b/lutrisweb/celery.py index c1997bec..5f366d55 100644 --- a/lutrisweb/celery.py +++ b/lutrisweb/celery.py @@ -9,8 +9,8 @@ os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'lutrisweb.settings.local') app = Celery('lutrisweb') -app.config_from_object('django.conf:settings') -app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) +app.config_from_object('django.conf:settings', namespace='CELERY') +app.autodiscover_tasks() @app.task(bind=True) diff --git a/lutrisweb/settings/base.py b/lutrisweb/settings/base.py index 6736828d..6c443e4a 100644 --- a/lutrisweb/settings/base.py +++ b/lutrisweb/settings/base.py @@ -82,6 +82,7 @@ def media_directory(path): TGD_LUTRIS_BANNER_PATH = media_directory('thegamesdb/lutris-banners') SECRET_KEY = os.environ.get('SECRET_KEY', 'changeme') +MOBY_API_KEY = os.environ.get('MOBY_API_KEY', 'Provide API key') MIDDLEWARE = [ # Caching disabled until proper invalidation is implemented @@ -228,6 +229,10 @@ def media_directory(path): 'clear-orphan-revisions': { 'task': 'games.tasks.clear_orphan_revisions', 'schedule': crontab(hour=17, minute=30) + }, + 'add-new-games': { + 'task': 'games.tasks.add_new_games', + 'schedule': crontab(minute=0, hour='*/3') } } BROKER_URL = 'amqp://guest:guest@localhost//'