diff --git a/core/admin.py b/core/admin.py index 3345fc2..58e13bf 100644 --- a/core/admin.py +++ b/core/admin.py @@ -2,11 +2,56 @@ from __future__ import unicode_literals from django.contrib import admin +from django.contrib.auth.admin import UserAdmin -# Register/Unregister models here. -from core.models import UserRepo, Issue, IssueLabel +from core.models import UserRepo, Issue, IssueLabel, Region, RegionAdmin from django.contrib.auth.models import * -admin.site.unregister(Group) -admin.site.unregister(User) -admin.site.register(UserRepo) + +class UserRepoAdmin(admin.ModelAdmin): + """Used to alter `UserRepo` admin site.""" + fieldsets = ( + (None, { + 'fields': ('user', 'repo', 'regions') + }), + ) + + def formfield_for_manytomany(self, db_field, request, **kwargs): + """Only show regions related to logged in user when filling `userrepo` form""" + if db_field.name == "regions": + kwargs["queryset"] = Region.objects.filter(regionadmin=request.user) + return super(UserRepoAdmin, self).formfield_for_manytomany(db_field, request, **kwargs) + + def save_form(self, request, form, change): + """Automatically fills author by extracting it from currunt login user.""" + obj = super( UserRepoAdmin, self).save_form(request, form, change) + if not change: + obj.author = request.user + return obj + + def get_queryset(self, request): + """Only let the user view their own `UserRepos`.""" + qs = super(UserRepoAdmin, self).get_queryset(request) + if request.user.is_superuser: + return qs + return qs.filter(author=request.user) + + def save_model(self, request, obj, form, change): + """Save Model""" + obj.author = request.user + super(UserRepoAdmin, self).save_model(request, obj, form, change) + + def has_change_permission(self, request, obj=None): + """Only give the user permissions to modify their own `UserRepos`.""" + if not obj: + return True + return obj.author == request.user or request.user.is_superuser + + +# Re-register UserAdmin +admin.site.register(User, UserAdmin) +admin.site.register(RegionAdmin) + +admin.site.register(UserRepo, UserRepoAdmin) +admin.site.register(Region) +admin.site.register(Issue) diff --git a/core/migrations/0001_initial.py b/core/migrations/0001_initial.py deleted file mode 100644 index d7c9406..0000000 --- a/core/migrations/0001_initial.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.1 on 2017-05-31 06:03 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name='UserRepo', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('user', models.CharField(max_length=100)), - ('repo', models.CharField(max_length=100)), - ('created', models.DateTimeField(auto_now_add=True)), - ], - options={ - 'ordering': ('created',), - }, - ), - ] diff --git a/core/migrations/0002_auto_20170601_0658.py b/core/migrations/0002_auto_20170601_0658.py deleted file mode 100644 index 60ef7f0..0000000 --- a/core/migrations/0002_auto_20170601_0658.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.1 on 2017-06-01 06:58 -from __future__ import unicode_literals - -from django.db import migrations, models -import django.db.models.deletion -import uuid - - -class Migration(migrations.Migration): - - dependencies = [ - ('core', '0001_initial'), - ] - - operations = [ - migrations.CreateModel( - name='Issue', - fields=[ - ('issue_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('title', models.CharField(max_length=100)), - ('experience_needed', models.CharField(choices=[(0, 'Easyfix'), (1, 'Moderate'), (2, 'Senior')], default=1, max_length=10)), - ('expected_time', models.CharField(max_length=100)), - ('language', models.CharField(max_length=100)), - ('tech_stack', models.CharField(max_length=100)), - ('created_at', models.DateTimeField()), - ('updated_at', models.DateTimeField()), - ('issue_details', models.TextField()), - ('issue_number', models.IntegerField()), - ('issue_url', models.CharField(max_length=2000)), - ], - ), - migrations.CreateModel( - name='IssueLabel', - fields=[ - ('label_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('label_url', models.CharField(max_length=2000)), - ('label_name', models.CharField(max_length=100)), - ('label_color', models.CharField(max_length=10)), - ], - ), - migrations.AddField( - model_name='issue', - name='issue_labels', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='core.IssueLabel'), - ), - ] diff --git a/core/migrations/__init__.py b/core/migrations/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/core/models.py b/core/models.py index 2613e78..592c024 100644 --- a/core/models.py +++ b/core/models.py @@ -3,13 +3,30 @@ from datetime import timedelta from django.db import models -from core.utils.services import request_github_issues - -from celery.task.schedules import crontab +from django.contrib.auth.models import AbstractUser from celery.decorators import periodic_task +from core.utils.services import request_github_issues + ISSUE_UPDATE_PERIOD = 15 # in minutes + +class Region(models.Model): + """Used to store data for different regions.""" + region_name = models.CharField(max_length=100, unique=True) + region_image = models.URLField(blank=True) + + class Meta: + ordering = ('region_name',) # Ascending order according to region name. + + def __str__(self): + return '%s' % (self.region_name) + + +class RegionAdmin(AbstractUser): + regions = models.ManyToManyField(Region) + + class UserRepo(models.Model): """ UserRepo model is used to store the username and repo-name @@ -17,11 +34,16 @@ class UserRepo(models.Model): """ user = models.CharField(max_length=100) repo = models.CharField(max_length=100) + author = models.ForeignKey(RegionAdmin) + regions = models.ManyToManyField(Region) created = models.DateTimeField(auto_now_add=True) class Meta: ordering = ('created',) # Ascending order according to date created. - unique_together = ("user", "repo") # Avoid repo duplicates. + unique_together = ("user", "repo", "author") # Avoid repo duplicates. + + def __str__(self): + return '/%s/%s' % (self.user, self.repo) class IssueLabel(models.Model): @@ -32,7 +54,7 @@ class IssueLabel(models.Model): label_url = models.URLField() label_name = models.CharField(max_length=100) label_color = models.CharField(max_length=6) - + class Meta: ordering = ('label_name',) # Ascending order according to label_name. @@ -67,7 +89,8 @@ class Issue(models.Model): issue_labels = models.ManyToManyField(IssueLabel, blank=True) issue_url = models.URLField() issue_body = models.TextField() - + regions = models.ManyToManyField(Region) + class Meta: ordering = ('updated_at',) # Ascending order according to updated_at. @@ -78,32 +101,56 @@ def periodic_issues_updater(): Update `Issue` model in the database in every `ISSUE_UPDATE_PERIOD` minutes. """ - list_of_repos = UserRepo.objects.values('user', 'repo',) + list_of_repos = UserRepo.objects.values('id', 'user', 'repo',) for repo in list_of_repos: + region_queryset = retrive_regions_for_a_user(repo['id']) issue_list = request_github_issues(repo['user'], repo['repo']) if issue_list['error']: print "Error" + str(issue_list['data']) else: for issue in issue_list['data']: - validate_and_store_issue(issue) + validate_and_store_issue(issue, region_queryset) + +def retrive_regions_for_a_user(user_repo_id): + """Fetches all the regions related to a user.""" + region_queryset = Region.objects.filter(userrepo=user_repo_id) + return region_queryset -def validate_and_store_issue(issue): +def validate_and_store_issue(issue, region_queryset): """ Validate issue:- if valid - store it into database, else - Do not store in database """ - if issue['state'] == 'open': - experience_needed, language, expected_time, technology_stack = parse_issue(issue['body']) + if is_issue_state_open(issue): + if is_issue_valid(issue): + store_issue_in_db(issue, region_queryset) - if experience_needed and language and expected_time and technology_stack: - store_issue_in_db(issue, experience_needed, language, expected_time, technology_stack) - else: - print 'Issue with id ' + str(issue['id']) + ' is not valid for our system.' +def is_issue_state_open(issue): + """ + Returns true if issue state is open else + return false and delete the issue from database. + """ + if issue['state'] == 'open': + return True else: delete_closed_issues(issue) # Delete closed issues from db. + return False -def store_issue_in_db(issue, experience_needed, language, expected_time, technology_stack): +def is_issue_valid(issue): + """ + Checks if issue is valid for system or not. + Return True if valid else return false. + """ + parsed = parse_issue(issue['body']) + for item in parsed: + if not item: + return False # issue is not valid + print 'Issue with id ' + str(issue['id']) + ' is not valid for our system.' + return True # issue is valid + +def store_issue_in_db(issue, region_queryset): """Stores issue in db""" + experience_needed, language, expected_time, technology_stack = parse_issue(issue['body']) experience_needed = experience_needed.strip().lower() language = language.strip().lower() expected_time = expected_time.strip().lower() @@ -120,6 +167,7 @@ def store_issue_in_db(issue, experience_needed, language, expected_time, technol label_url=label['url'], label_color=label['color']) label_instance.save() issue_instance.issue_labels.add(label_instance) + issue_instance.regions.add(*region_queryset) def delete_closed_issues(issue): """Delete issues that are closed on GitHub but present in our db""" diff --git a/core/serializers.py b/core/serializers.py index 7d380b0..08e01d4 100644 --- a/core/serializers.py +++ b/core/serializers.py @@ -1,5 +1,5 @@ from rest_framework import serializers -from core.models import UserRepo, Issue, IssueLabel +from core.models import UserRepo, Issue, IssueLabel, Region class UserRepoSerializer(serializers.ModelSerializer): @@ -20,6 +20,15 @@ class Meta: fields = ('label_id', 'label_name', 'label_color', 'label_url',) +class RegionSerializer(serializers.ModelSerializer): + """ + Serializer for `Region` Model. + """ + class Meta: + model = Region + fields = ('id','region_name', 'region_image',) + + class IssueSerializer(serializers.ModelSerializer): """ Serializer for `Issue` Model. @@ -30,4 +39,4 @@ class Meta: model = Issue fields = ('issue_id', 'title', 'experience_needed', 'expected_time', 'language', 'tech_stack', 'created_at', 'updated_at', - 'issue_number', 'issue_labels', 'issue_url', 'issue_body') + 'issue_number', 'issue_labels', 'issue_url', 'issue_body', 'regions',) diff --git a/core/tests.py b/core/tests.py index 54e50da..483c41b 100644 --- a/core/tests.py +++ b/core/tests.py @@ -8,12 +8,15 @@ from django.test import TestCase from rest_framework.test import APIClient from rest_framework import status +from requests.exceptions import ConnectionError -from .models import UserRepo, parse_issue, validate_and_store_issue, Issue, delete_closed_issues +from .models import (UserRepo, parse_issue, validate_and_store_issue, Issue, delete_closed_issues, + is_issue_valid, is_issue_state_open, periodic_issues_updater, Region, RegionAdmin, + retrive_regions_for_a_user) from .utils.mock_api import api_response_issues from .utils.services import request_github_issues -SAMPLE_ISSUE = { +SAMPLE_VALID_ISSUE = { "html_url": "https://github.com/mozillacampusclubs/issue_parser_backend/issues/7", "id": 233564738, "number": 7, @@ -44,7 +47,10 @@ def setUp(self): """Define the test client and other test variables.""" self.user = 'razat249' self.repo = 'github-view' - self.user_repo = UserRepo(user=self.user, repo=self.repo) + self.author = RegionAdmin.objects.create_user( + username='jacob', password='top_secret' + ) + self.user_repo = UserRepo(user=self.user, repo=self.repo, author=self.author) def test_user_repo_model_can_create_a_userrepo(self): """Test the `UserRepo` model can create a `user_repo`.""" @@ -61,60 +67,137 @@ def test_user_repo_model_can_delete_a_userrepo(self): new_count = UserRepo.objects.count() self.assertEqual(old_count, new_count) +class RegionModelTestCase(TestCase): + """This class defines the test suite for the `Region` model.""" + + def setUp(self): + """Define the test client and other test variables.""" + self.region_name = 'Mozilla India' + self.region_image = 'https://example.com/image.jpg' + self.region_instance = Region(region_name=self.region_name, region_image=self.region_image) + + def test_region_model_can_create_region(self): + """Tests `Region` model can create Regions""" + old_count = Region.objects.count() + self.region_instance.save() + new_count = Region.objects.count() + self.assertNotEqual(old_count, new_count) + + def test_region_model_can_delete_region(self): + """Tests `Region` model can delete Regions""" + old_count = Region.objects.count() + self.region_instance.save() + self.region_instance.delete() + new_count = Region.objects.count() + self.assertEqual(old_count, new_count) class IssueModelAndFetcherTestCase(TestCase): """This class defines the test suite for the `issue fetcher` component.""" def setUp(self): """Initial setup for running tests.""" - pass + self.USER_ID = 1 + self.USER_REPO_ID = 1 + self.author = RegionAdmin.objects.create_user( + id=self.USER_ID, username='jacob', password='top_secret' + ) + self.region = Region(region_name="Mozilla India") + self.region.save() + self.user_repo = UserRepo(id=self.USER_REPO_ID, user='razat249', repo='github-view', author=self.author) + self.user_repo.save() + self.user_repo.regions.add(self.region) + self.region_queryset = Region.objects.filter(userrepo=self.USER_ID) def test_api_can_request_issues(self): """Test the request function""" payload = request_github_issues('razat249', 'github-view') - self.assertEqual(payload['error'], False) - self.assertLess(payload['status_code'], 400) + if payload['error_type'] == ConnectionError: + pass + else: + self.assertEqual(payload['error'], False) + self.assertLess(payload['status_code'], 400) def test_api_request_can_handle_errors(self): """Test the request function can handle errors""" # wrong repo name to test error handling. payload = request_github_issues('razat249', 'wrong_repo') - self.assertEqual(payload['error'], True) - self.assertGreaterEqual(payload['status_code'], 400) + if payload['error_type'] == ConnectionError: + pass + else: + self.assertEqual(payload['error'], True) + self.assertGreaterEqual(payload['status_code'], 400) def test_correct_issue_parsing(self): """Test for correct parsing of issues""" - issue = SAMPLE_ISSUE.copy() + issue = SAMPLE_VALID_ISSUE.copy() parsed = parse_issue(issue['body']) for item in parsed: self.assertTrue(item) + def test_issue_valid_and_not_valid_cases(self): + """Test for checking if issue is valid or not""" + valid_issue = SAMPLE_VALID_ISSUE.copy() + invalid_issue = valid_issue.copy() + invalid_issue['body'] = '' + self.assertTrue(is_issue_valid(valid_issue)) + self.assertFalse(is_issue_valid(invalid_issue)) + + def test_issue_state_open_or_not(self): + """Test for checking issue state""" + open_issue = SAMPLE_VALID_ISSUE.copy() + closed_issue = SAMPLE_VALID_ISSUE.copy() + closed_issue['state'] = 'closed' + self.assertTrue(is_issue_state_open(open_issue)) + self.assertFalse(is_issue_state_open(closed_issue)) + def test_validate_and_store_issue(self): """Test for validating and storing issues.""" old_count = Issue.objects.count() - validate_and_store_issue(SAMPLE_ISSUE) + validate_and_store_issue(SAMPLE_VALID_ISSUE, self.region_queryset) new_count = Issue.objects.count() - self.assertNotEqual(old_count, new_count) + self.assertLess(old_count, new_count) def test_api_can_delete_closed_issues_in_db(self): """Test for checking if issues are deleted when closed online but present in db""" - issue = SAMPLE_ISSUE.copy() - validate_and_store_issue(issue) + issue = SAMPLE_VALID_ISSUE.copy() + validate_and_store_issue(issue, self.region_queryset) issue['state'] = 'closed' old_count = Issue.objects.count() delete_closed_issues(issue) new_count = Issue.objects.count() self.assertLess(new_count, old_count) + def test_retrive_regions_for_a_user(self): + """Test function can retrive regions for a user.""" + regions = retrive_regions_for_a_user(self.USER_REPO_ID) + self.assertEqual(regions[0], self.region_queryset[0]) class ViewTestCase(TestCase): """This class defines the test suite for the api views.""" def setUp(self): """Define the test client and other test variables.""" + self.mock_regions = ["a", "n", "e", "b", "d", "c"] self.client = APIClient() + region_queryset = Region.objects.all() for issue in api_response_issues: - validate_and_store_issue(issue) + validate_and_store_issue(issue, region_queryset) + + def test_api_can_get_region_list(self): + """Test the api can get given region list.""" + response = self.client.get('/regionlist/', format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + + def test_api_can_get_region_list_ordered_by_name(self): + """Test the api gives list of regions in accessending order.""" + for s in self.mock_regions: + region_list = Region(region_name=s) + region_list.save() + response = self.client.get('/regionlist/', format="json") + response_content = json.loads(response.content) + sorted_mock_regions = sorted(self.mock_regions) + for i in xrange(len(sorted_mock_regions)): + self.assertEqual(sorted_mock_regions[i], response_content[i]['region_name']) def test_api_can_get_metadata(self): """Test the api can get given metadata.""" diff --git a/core/utils/services.py b/core/utils/services.py index a1b5336..2a1fcbc 100644 --- a/core/utils/services.py +++ b/core/utils/services.py @@ -3,14 +3,20 @@ """ import requests +from requests.exceptions import ConnectionError def request_github_issues(user, repo): """ Returns a list of all the issues of a repository in `json` format. """ - api_data = 'https://api.github.com/repos/'+ user +'/' + repo + '/issues?state=all' - response = requests.get(api_data) - if response.status_code < 400: - return {'error': False, 'data': response.json(), 'status_code': response.status_code} - else: - return {'error': True, 'data': response.json(), 'status_code': response.status_code} + try: + api_data = 'https://api.github.com/repos/'+ user +'/' + repo + '/issues?state=all' + response = requests.get(api_data) + if response.status_code < 400: + return {'error': False, 'error_type':None, 'data': response.json(), + 'status_code': response.status_code} + else: + return {'error': True, 'error_type':None, 'data': response.json(), + 'status_code': response.status_code} + except ConnectionError: + return {'error': True, 'error_type':ConnectionError, 'data': 'No Internet connection'} diff --git a/core/views.py b/core/views.py index 6aa5c28..0124d36 100644 --- a/core/views.py +++ b/core/views.py @@ -1,7 +1,7 @@ -from core.models import UserRepo, Issue -from core.serializers import UserRepoSerializer, IssueSerializer +from core.models import UserRepo, Issue, Region +from core.serializers import UserRepoSerializer, IssueSerializer, RegionSerializer from rest_framework import generics -from django_filters.rest_framework import DjangoFilterBackend +from django_filters.rest_framework import DjangoFilterBackend from rest_framework.filters import OrderingFilter from rest_framework.views import APIView from rest_framework.renderers import JSONRenderer, BrowsableAPIRenderer @@ -18,6 +18,14 @@ class UserRepoList(generics.ListAPIView): filter_fields = ('repo', 'user',) +class RegionList(generics.ListAPIView): + """ + Returns a list of regions. + """ + queryset = Region.objects.all() + serializer_class = RegionSerializer + + class IssueList(generics.ListAPIView): """ Returns a list of issues, by optionally filtering against @@ -27,7 +35,7 @@ class IssueList(generics.ListAPIView): queryset = Issue.objects.all() serializer_class = IssueSerializer filter_backends = (DjangoFilterBackend, OrderingFilter,) - filter_fields = ('language', 'tech_stack', 'experience_needed', 'expected_time',) + filter_fields = ('language', 'tech_stack', 'experience_needed', 'expected_time', 'regions',) ordering_fields = ('experience_needed', 'expected_time') diff --git a/issue_parser/settings.py b/issue_parser/settings.py index 526c24f..1066fbf 100644 --- a/issue_parser/settings.py +++ b/issue_parser/settings.py @@ -50,6 +50,8 @@ 'django_nose', ] +AUTH_USER_MODEL = 'core.RegionAdmin' + # Use nose to run all tests TEST_RUNNER = 'django_nose.NoseTestSuiteRunner' diff --git a/issue_parser/urls.py b/issue_parser/urls.py index 1b6d9ba..83201c1 100644 --- a/issue_parser/urls.py +++ b/issue_parser/urls.py @@ -19,6 +19,7 @@ from django.contrib import admin urlpatterns = [ + url(r'^regionlist/$', views.RegionList.as_view()), url(r'^userrepos/$', views.UserRepoList.as_view()), url(r'^issues/$', views.IssueList.as_view()), url(r'^metadata/$', views.MetaData.as_view()),