FEATURE: REST API (#34)

* FEATURE: REST API

* REFACTOR: disable nexted write

* FEATURE: add owner filter - wip

* FEATURE: add repo events and statistics - wip

* FIX: add repo size unit to statistic

* FEATURE: add openapi schema

* FEATURE: update requirements

* FEATURE: API

* FIX: update deps

* STYLE: fix pep

* FIX: move migrations

* FEATURE: bump release

Co-authored-by: Benjamin Pereto <[email protected]>

.pylintrc

@@ -33,6 +33,7 @@ limit-inference-results=100
 # List of plugins (as comma separated values of python module names) to load,
 # usually to register additional checkers.
 load-plugins=pylint_django
+django-settings-module=core.settings
 
 # Pickle collected data for later comparisons.
 persistent=yes
@@ -144,7 +145,8 @@ disable=print-statement,
         wildcard-import,
         missing-module-docstring,
         invalid-name,
-        too-many-ancestors
+        too-many-ancestors,
+        imported-auth-user
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

borg/requirements.in

@@ -0,0 +1,2 @@
+borgbackup==1.1.14
+mysql-connector==2.2.9

borg/requirements.txt

@@ -2,13 +2,7 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile borg/requirements.txt
+#    pip-compile
 #
-
-# install from github until issue https://github.com/bpereto/borg-hive/issues/19 is resolved by borgbackup release 1.1.14
-git+https://github.com/borgbackup/borg@c53190176389dcbd711060a8a9f4326e66d1d533#egg=borgbackup
-
-environs==7.4.0           # via -r borg/requirements.txt
-marshmallow==3.5.2        # via environs
-mysql-connector==2.2.9    # via -r borg/requirements.txt
-python-dotenv==0.13.0     # via environs
+borgbackup==1.1.14        # via -r requirements.in
+mysql-connector==2.2.9    # via -r requirements.in

docker-compose.dev.yml

@@ -39,7 +39,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfile.base
-    entrypoint: celery worker -A core -l DEBUG -B --scheduler django_celery_beat.schedulers:DatabaseScheduler
+    entrypoint: celery -A core worker -l DEBUG -B --scheduler django_celery_beat.schedulers:DatabaseScheduler
     environment:
       - DEBUG=True
       - MYSQL_HOST=db

docker-compose.yml

@@ -10,7 +10,7 @@ services:
       - .env
 
   app:
-    image: bpereto/borg-hive:0.1.2
+    image: bpereto/borg-hive:0.1.3
     env_file:
       - .env
     volumes:
@@ -23,8 +23,8 @@ services:
       - db
 
   worker:
-    image: bpereto/borg-hive:0.1.2
-    entrypoint: celery worker -A core -l INFO -B --scheduler django_celery_beat.schedulers:DatabaseScheduler
+    image: bpereto/borg-hive:0.1.3
+    entrypoint: celery -A core worker -l INFO -B --scheduler django_celery_beat.schedulers:DatabaseScheduler
     env_file:
       - .env
     volumes:
@@ -34,7 +34,7 @@ services:
       - db
 
   watcher:
-    image: bpereto/borg-hive:0.1.2
+    image: bpereto/borg-hive:0.1.3
     entrypoint: /bin/bash -c "/app/manage.py watch_repositories"
     env_file:
       - .env
@@ -45,7 +45,7 @@ services:
     restart: 'on-failure'
 
   borg:
-    image: bpereto/borg-hive:borg-0.1.2
+    image: bpereto/borg-hive:borg-0.1.3
     depends_on:
       - db
     env_file:

requirements-dev.txt

@@ -1,31 +1,6 @@
-#
-# This file is autogenerated by pip-compile
-# To update, run:
-#
-#    pip-compile requirements-dev.txt
-#
-astroid==2.4.1            # via -r requirements-dev.txt, pylint
-certifi==2020.4.5.1       # via -r requirements-dev.txt
-chardet==3.0.4            # via -r requirements-dev.txt
-commonmark==0.9.1         # via -r requirements-dev.txt
-coverage==5.1             # via -r requirements-dev.txt
-docutils==0.16            # via -r requirements-dev.txt
-idna==2.9                 # via -r requirements-dev.txt
-isort==4.3.21             # via -r requirements-dev.txt, pylint
-lazy-object-proxy==1.4.3  # via -r requirements-dev.txt, astroid
-markupsafe==1.1.1         # via -r requirements-dev.txt
-mccabe==0.6.1             # via -r requirements-dev.txt, pylama, pylint
-pycodestyle==2.5.0        # via -r requirements-dev.txt, pylama
-pydocstyle==5.0.2         # via -r requirements-dev.txt, pylama
-pyflakes==2.2.0           # via -r requirements-dev.txt, pylama
-pylama==7.7.1             # via -r requirements-dev.txt
-pylint-django==2.0.15     # via -r requirements-dev.txt
-pylint-plugin-utils==0.6  # via -r requirements-dev.txt, pylint-django
-pylint==2.5.2             # via -r requirements-dev.txt, pylint-django, pylint-plugin-utils
-pyparsing==2.4.7          # via -r requirements-dev.txt
-pytz==2020.1              # via -r requirements-dev.txt
-six==1.14.0               # via -r requirements-dev.txt, astroid
-snowballstemmer==2.0.0    # via pydocstyle
-toml==0.10.0              # via -r requirements-dev.txt, pylint
-urllib3==1.25.9           # via -r requirements-dev.txt
-wrapt==1.12.1             # via -r requirements-dev.txt, astroid
+pylint
+astroid
+pylama
+pylint-django
+pylint-plugin-utils
+coverage

requirements.in

@@ -0,0 +1,19 @@
+celery==5.0.5
+django-celery-beat==2.1.0
+django-crispy-forms==1.10.0
+django-extensions==3.1.0
+django-ldapdb==1.5.1
+django-login-required-middleware==0.5.0
+django-polymorphic==3.0.0
+django==3.1.4
+djangorestframework-queryfields==1.0.0
+djangorestframework==3.12.2
+environs==9.2.0
+inotify==0.2.10
+mysqlclient==2.0.2
+pyyaml==5.3.1
+redis==3.5.3
+requests==2.25.1
+rules==2.2
+sshpubkeys==3.1.0
+uritemplate==3.0.1

requirements.txt

@@ -2,47 +2,55 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile requirements.txt
+#    pip-compile requirements.in
 #
-amqp==2.5.2               # via -r requirements.txt, kombu
-asgiref==3.2.7            # via -r requirements.txt, django
-billiard==3.6.3.0         # via -r requirements.txt, celery
-celery==4.4.2             # via -r requirements.txt, django-celery-beat
-certifi==2020.4.5.1       # via -r requirements.txt, requests
-cffi==1.14.0              # via -r requirements.txt, cryptography
-chardet==3.0.4            # via -r requirements.txt, requests
-cryptography==3.2       # via -r requirements.txt, sshpubkeys
-django-celery-beat==2.0.0  # via -r requirements.txt
-django-crispy-forms==1.9.0  # via -r requirements.txt
-django-extensions==2.2.9  # via -r requirements.txt
-django-ldapdb==1.4.0      # via -r requirements.txt
-django-login-required-middleware==0.5.0  # via -r requirements.txt
-django-polymorphic==2.1.2  # via -r requirements.txt
-django-timezone-field==4.0  # via -r requirements.txt, django-celery-beat
-django==3.0.7             # via -r requirements.txt, django-celery-beat, django-ldapdb, django-polymorphic, django-timezone-field, djangorestframework
-djangorestframework==3.11.0  # via -r requirements.txt
-ecdsa==0.15               # via -r requirements.txt, sshpubkeys
-environs==7.4.0           # via -r requirements.txt
-idna==2.9                 # via -r requirements.txt, requests
-inotify==0.2.10           # via -r requirements.txt
-kombu==4.6.8              # via -r requirements.txt, celery
-marshmallow==3.5.2        # via -r requirements.txt, environs
-mysqlclient==1.4.6        # via -r requirements.txt
-nose==1.3.7               # via -r requirements.txt, inotify
-pyasn1-modules==0.2.8     # via -r requirements.txt, python-ldap
-pyasn1==0.4.8             # via -r requirements.txt, pyasn1-modules, python-ldap
-pycparser==2.20           # via -r requirements.txt, cffi
-python-crontab==2.4.2     # via -r requirements.txt, django-celery-beat
-python-dateutil==2.8.1    # via -r requirements.txt, python-crontab
-python-dotenv==0.13.0     # via -r requirements.txt, environs
-python-ldap==3.2.0        # via -r requirements.txt, django-ldapdb
-pytz==2020.1              # via -r requirements.txt, celery, django, django-timezone-field
-pyyaml==5.3.1             # via -r requirements.txt
-redis==3.5.0              # via -r requirements.txt
-requests==2.23.0          # via -r requirements.txt
-rules==2.2                # via -r requirements.txt
-six==1.14.0               # via -r requirements.txt, cryptography, django-extensions, ecdsa, python-dateutil
-sqlparse==0.3.1           # via -r requirements.txt, django
-sshpubkeys==3.1.0         # via -r requirements.txt
-urllib3==1.25.9           # via -r requirements.txt, requests
-vine==1.3.0               # via -r requirements.txt, amqp, celery
+amqp==5.0.2               # via kombu
+asgiref==3.3.1            # via django
+billiard==3.6.3.0         # via celery
+celery==5.0.5             # via -r requirements.in, django-celery-beat
+certifi==2020.12.5        # via requests
+cffi==1.14.4              # via cryptography
+chardet==4.0.0            # via requests
+click-didyoumean==0.0.3   # via celery
+click-plugins==1.1.1      # via celery
+click-repl==0.1.6         # via celery
+click==7.1.2              # via celery, click-didyoumean, click-plugins, click-repl
+cryptography==3.3.1       # via sshpubkeys
+django-celery-beat==2.1.0  # via -r requirements.in
+django-crispy-forms==1.10.0  # via -r requirements.in
+django-extensions==3.1.0  # via -r requirements.in
+django-ldapdb==1.5.1      # via -r requirements.in
+django-login-required-middleware==0.5.0  # via -r requirements.in
+django-polymorphic==3.0.0  # via -r requirements.in
+django-timezone-field==4.1.1  # via django-celery-beat
+django==3.1.4             # via -r requirements.in, django-celery-beat, django-ldapdb, django-polymorphic, django-timezone-field, djangorestframework
+djangorestframework-queryfields==1.0.0  # via -r requirements.in
+djangorestframework==3.12.2  # via -r requirements.in
+ecdsa==0.16.1             # via sshpubkeys
+environs==9.2.0           # via -r requirements.in
+idna==2.10                # via requests
+inotify==0.2.10           # via -r requirements.in
+kombu==5.0.2              # via celery
+marshmallow==3.10.0       # via environs
+mysqlclient==2.0.2        # via -r requirements.in
+nose==1.3.7               # via inotify
+prompt-toolkit==3.0.8     # via click-repl
+pyasn1-modules==0.2.8     # via python-ldap
+pyasn1==0.4.8             # via pyasn1-modules, python-ldap
+pycparser==2.20           # via cffi
+python-crontab==2.5.1     # via django-celery-beat
+python-dateutil==2.8.1    # via python-crontab
+python-dotenv==0.15.0     # via environs
+python-ldap==3.3.1        # via django-ldapdb
+pytz==2020.4              # via celery, django, django-timezone-field
+pyyaml==5.3.1             # via -r requirements.in
+redis==3.5.3              # via -r requirements.in
+requests==2.25.1          # via -r requirements.in
+rules==2.2                # via -r requirements.in
+six==1.15.0               # via click-repl, cryptography, ecdsa, python-dateutil
+sqlparse==0.4.1           # via django
+sshpubkeys==3.1.0         # via -r requirements.in
+uritemplate==3.0.1        # via -r requirements.in
+urllib3==1.26.2           # via requests
+vine==5.0.0               # via amqp, celery
+wcwidth==0.2.5            # via prompt-toolkit

src/api/__init__.py

@@ -0,0 +1 @@
+default_app_config = 'api.apps.ApiConfig'

src/api/apps.py

@@ -0,0 +1,8 @@
+from django.apps import AppConfig
+
+
+class ApiConfig(AppConfig):
+    """
+    django app config for drf api
+    """
+    name = 'api'

src/api/lib/serializers.py

@@ -0,0 +1,94 @@
+'''
+Definition of the REST framework serializers.
+'''
+
+__all__ = (
+    'SimpleModelSerializer',
+    'SimpleHyperlinkedModelSerializer',
+)
+
+from drf_queryfields import QueryFieldsMixin
+from rest_framework.serializers import (HyperlinkedModelSerializer,
+                                        HyperlinkedRelatedField,
+                                        ModelSerializer, ReadOnlyField)
+from rest_framework.utils.field_mapping import get_nested_relation_kwargs
+
+# pylint: disable=redefined-builtin
+
+NAMESPACE = 'api'
+
+
+class SimpleModelSerializer(QueryFieldsMixin, ModelSerializer):  # pylint: disable=too-few-public-methods
+    '''
+    A REST framework `ModelSerializer` on dope, which automatically initializes
+    the Meta class of the serializer, based on the arguments provided in the
+    constructor.
+    '''
+    namespace = NAMESPACE
+
+    def __init__(self, *args, model=None, fields=None, **kwargs):
+        '''
+        Initialize the Meta class before initializing the serializer.
+        '''
+
+        if not hasattr(self, 'Meta'):
+            self.Meta = type('Meta', (), {'model': model, 'fields': fields})  # pylint: disable=invalid-name
+
+        super().__init__(*args, **kwargs)
+
+    def build_url_field(self, field_name, model_class):
+        '''
+        Create a field representing the object's own URL.
+
+        Please note this method is overloaded to add the namespace to the view
+        name.
+        '''
+        field_class, field_kwargs = super().build_url_field(field_name, model_class)
+        field_kwargs['view_name'] = '{}:{}'.format(self.namespace, field_kwargs['view_name'])
+        return field_class, field_kwargs
+
+    def build_relational_field(self, field_name, relation_info):
+        field_class, field_kwargs = super().build_relational_field(field_name, relation_info)
+
+        if hasattr(field_kwargs, 'view_name'):
+            field_kwargs['view_name'] = '{}:{}'.format(self.namespace, field_kwargs['view_name'])
+
+        return field_class, field_kwargs
+
+
+class SimpleHyperlinkedRelatedField(HyperlinkedRelatedField):
+    """
+    A REST framework `HyperlinkedRelatedField` to support namespaces
+    """
+
+    namespace = NAMESPACE
+
+    def __init__(self, *args, **kwargs):
+        kwargs['view_name'] = self.namespace + ':' + kwargs['view_name']
+        super().__init__(*args, **kwargs)
+
+
+class SimpleHyperlinkedModelSerializer(SimpleModelSerializer, HyperlinkedModelSerializer):  # pylint: disable=too-few-public-methods
+    '''
+    A REST framework `HyperlinkedModelSerializer` on dope, which automatically
+    initializes the Meta class of the serializer, based on the arguments
+    provided in the constructor.
+    '''
+    serializer_related_field = SimpleHyperlinkedRelatedField
+
+    id = ReadOnlyField()
+
+    def build_nested_field(self, field_name, relation_info, nested_depth):
+        """
+        Create nested fields for forward and reverse relationships.
+        """
+        class NestedSerializer(SimpleHyperlinkedModelSerializer):  # pylint: disable=too-few-public-methods,missing-docstring
+            class Meta:  # pylint: disable=too-few-public-methods,missing-docstring
+                model = relation_info.related_model
+                depth = nested_depth - 1
+                fields = '__all__'
+
+        field_class = NestedSerializer
+        field_kwargs = get_nested_relation_kwargs(relation_info)
+
+        return field_class, field_kwargs