diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index d86e3124..0041c58d 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -10,6 +10,3 @@ ENV PYTHONUNBUFFERED 1 # [Optional] Uncomment this section to install additional OS packages. # RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ # && apt-get -y install --no-install-recommends - - - diff --git a/.github/workflows/linter.yaml b/.github/workflows/linter.yaml new file mode 100644 index 00000000..aa225e94 --- /dev/null +++ b/.github/workflows/linter.yaml @@ -0,0 +1,32 @@ +name: Lint Code Base +on: + pull_request: + branches: + - master +jobs: + build: + if: github.repository == 'coronasafe/ayushma' + name: Lint Code Base + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + statuses: write + steps: + - name: Checkout Code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Lint Code Base + uses: super-linter/super-linter/slim@v6 + env: + DEFAULT_BRANCH: master + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VALIDATE_ALL_CODEBASE: false + VALIDATE_PYTHON_BLACK: true + VALIDATE_PYTHON_FLAKE8: true + VALIDATE_PYTHON_ISORT: true + LINTER_RULES_PATH: / + PYTHON_FLAKE8_CONFIG_FILE: "setup.cfg" + PYTHON_BLACK_CONFIG_FILE: "pyproject.toml" + PYTHON_ISORT_CONFIG_FILE: "pyproject.toml" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2b9044e1..3680aa4a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,26 +1,26 @@ exclude: "docs|node_modules|migrations|.git|.venv" default_stages: [commit] fail_fast: true - repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.5.0 hooks: - id: check-yaml - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/PyCQA/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort - + additional_dependencies: ["isort[pyproject]"] - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 24.3.0 hooks: - id: black - + args: ["--config=pyproject.toml"] - repo: https://github.com/PyCQA/flake8 - rev: 4.0.1 + rev: 7.0.0 hooks: - id: flake8 args: ["--config=setup.cfg"] + additional_dependencies: [flake8-isort] diff --git a/.vscode/settings.json b/.vscode/settings.json index 02714a6b..2e6efc0c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -10,7 +10,7 @@ "[python]": { "editor.defaultFormatter": "ms-python.black-formatter", "editor.codeActionsOnSave": { - "source.organizeImports": true + "source.organizeImports": "explicit" } }, "editor.formatOnSave": true, diff --git a/Makefile b/Makefile index 209e88c2..a9548416 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ logs: makemigrations: up docker exec django bash -c "python manage.py makemigrations" - + checkmigration: docker compose -f $(docker_config_file) exec django bash -c "python manage.py makemigrations --check --dry-run" diff --git a/ayushma/admin.py b/ayushma/admin.py index ceb68ae7..b86cd670 100644 --- a/ayushma/admin.py +++ b/ayushma/admin.py @@ -1,7 +1,6 @@ from django.contrib import admin from django.contrib.auth.admin import UserAdmin as BaseUserAdmin from djangoql.admin import DjangoQLSearchMixin -from simple_history.admin import SimpleHistoryAdmin from ayushma.models.services import Service, TempToken from ayushma.models.testsuite import Feedback, TestQuestion, TestRun, TestSuite @@ -32,7 +31,15 @@ class UserAdmin(DjangoQLSearchMixin, BaseUserAdmin): fieldsets = ( ( None, - {"fields": ("email", "username", "password", "external_id", "allow_key")}, + { + "fields": ( + "email", + "username", + "password", + "external_id", + "allow_key", + ) + }, ), ( "User info", diff --git a/ayushma/apps.py b/ayushma/apps.py index a0be6977..ae9c4b39 100644 --- a/ayushma/apps.py +++ b/ayushma/apps.py @@ -6,5 +6,5 @@ class AyushmaConfig(AppConfig): name = "ayushma" def ready(self): - #from .signals import # noqa: F401 + # from .signals import # noqa: F401 pass diff --git a/ayushma/management/commands/upsert.py b/ayushma/management/commands/upsert.py index a239f19e..69694318 100644 --- a/ayushma/management/commands/upsert.py +++ b/ayushma/management/commands/upsert.py @@ -31,7 +31,8 @@ class Command(BaseCommand): def handle(self, *args, **options): upsert_dir = "upsert" pinecone.init( - api_key=settings.PINECONE_API_KEY, environment=settings.PINECONE_ENVIRONMENT + api_key=settings.PINECONE_API_KEY, + environment=settings.PINECONE_ENVIRONMENT, ) print("Initialized Pinecone and OpenAI") @@ -49,7 +50,7 @@ def handle(self, *args, **options): batch_size = 100 # process everything in batches of 100 (creates 100 vectors per upset) - print(f"Fetching Pinecone index...") + print("Fetching Pinecone index...") if settings.PINECONE_INDEX not in pinecone.list_indexes(): pinecone.create_index( settings.PINECONE_INDEX, diff --git a/ayushma/models/services.py b/ayushma/models/services.py index 0f9db546..6b961eb9 100644 --- a/ayushma/models/services.py +++ b/ayushma/models/services.py @@ -1,5 +1,4 @@ import secrets -import time from django.db import models from django.utils import timezone diff --git a/ayushma/models/token.py b/ayushma/models/token.py index 5aab6450..b9a3452a 100644 --- a/ayushma/models/token.py +++ b/ayushma/models/token.py @@ -1,6 +1,6 @@ +from django.conf import settings from django.db import models from django.utils.translation import gettext_lazy as _ -from django.conf import settings from ayushma.token import RandomStringTokenGenerator from utils.models.base import BaseModel @@ -11,9 +11,7 @@ class ResetPasswordToken(BaseModel): settings.AUTH_USER_MODEL, related_name="password_reset_tokens", on_delete=models.CASCADE, - verbose_name=_( - "The User which is associated to this password reset token" - ), + verbose_name=_("The User which is associated to this password reset token"), ) created_at = models.DateTimeField( auto_now_add=True, verbose_name=_("When was this token generated") diff --git a/ayushma/permissions.py b/ayushma/permissions.py index a6005929..09166692 100644 --- a/ayushma/permissions.py +++ b/ayushma/permissions.py @@ -1,5 +1,3 @@ -from ipaddress import ip_address - from django.utils import timezone from rest_framework import permissions diff --git a/ayushma/serializers/document.py b/ayushma/serializers/document.py index 1d727445..bb57027d 100644 --- a/ayushma/serializers/document.py +++ b/ayushma/serializers/document.py @@ -1,12 +1,6 @@ -import os -import uuid - -from django.conf import settings -from django.core.files.storage import FileSystemStorage from rest_framework import serializers -from rest_framework.response import Response -from ayushma.models import Document, DocumentType +from ayushma.models import Document class DocumentSerializer(serializers.ModelSerializer): @@ -23,7 +17,12 @@ class Meta: "text_content", "uploading", ) - read_only_fields = ("external_id", "created_at", "modified_at", "uploading") + read_only_fields = ( + "external_id", + "created_at", + "modified_at", + "uploading", + ) write_only_fields = ("file",) def validate(self, data): diff --git a/ayushma/serializers/testsuite.py b/ayushma/serializers/testsuite.py index a9fc33f2..5e7e07d2 100644 --- a/ayushma/serializers/testsuite.py +++ b/ayushma/serializers/testsuite.py @@ -9,7 +9,7 @@ TestSuite, ) from ayushma.serializers.document import DocumentSerializer -from ayushma.serializers.project import ProjectSerializer, ProjectUpdateSerializer +from ayushma.serializers.project import ProjectUpdateSerializer from ayushma.serializers.users import UserSerializer @@ -67,7 +67,12 @@ class Meta: "modified_at", ) - read_only_fields = ("user_object", "external_id", "created_at", "modified_at") + read_only_fields = ( + "user_object", + "external_id", + "created_at", + "modified_at", + ) class TestResultSerializer(serializers.ModelSerializer): diff --git a/ayushma/token.py b/ayushma/token.py index ade3bd71..b95fbb6a 100644 --- a/ayushma/token.py +++ b/ayushma/token.py @@ -7,6 +7,4 @@ def __init__(self, length=6): self.length = length def generate(self): - return "".join( - [random.choice(string.digits) for _ in range(self.length)] - ) + return "".join([random.choice(string.digits) for _ in range(self.length)]) diff --git a/ayushma/utils/converse.py b/ayushma/utils/converse.py index 7063b18b..3fdf0f52 100644 --- a/ayushma/utils/converse.py +++ b/ayushma/utils/converse.py @@ -73,7 +73,7 @@ def converse_api( converse_type = "audio" if audio else "text" # convert stream to boolean - if type(stream) != bool: + if not isinstance(stream, bool): if stream == "false": stream = False else: @@ -82,7 +82,7 @@ def converse_api( if is_thread: stream = False # Threads do not support streaming - if type(generate_audio) != bool: + if not isinstance(generate_audio, bool): if generate_audio == "false": generate_audio = False else: @@ -195,7 +195,8 @@ def converse_api( response_message = list(response_message)[0] return Response( - ChatMessageSerializer(response_message).data, status=status.HTTP_200_OK + ChatMessageSerializer(response_message).data, + status=status.HTTP_200_OK, ) return response diff --git a/ayushma/views/auth.py b/ayushma/views/auth.py index bb7a78f6..ed64f40f 100644 --- a/ayushma/views/auth.py +++ b/ayushma/views/auth.py @@ -1,19 +1,14 @@ -from drf_spectacular.utils import ( - extend_schema, - OpenApiResponse, -) +from drf_spectacular.utils import OpenApiResponse, extend_schema from rest_framework import status -from rest_framework.viewsets import GenericViewSet -from rest_framework.decorators import action -from rest_framework.response import Response from rest_framework.authtoken.models import Token +from rest_framework.decorators import action from rest_framework.exceptions import ParseError from rest_framework.permissions import AllowAny +from rest_framework.response import Response +from rest_framework.viewsets import GenericViewSet from ayushma.serializers.auth import AuthSerializer -from ayushma.serializers.users import ( - UserCreateSerializer, -) +from ayushma.serializers.users import UserCreateSerializer class AuthViewSet(GenericViewSet): diff --git a/ayushma/views/chat.py b/ayushma/views/chat.py index c6f72163..9767e25e 100644 --- a/ayushma/views/chat.py +++ b/ayushma/views/chat.py @@ -3,7 +3,7 @@ from django.conf import settings from drf_spectacular.utils import extend_schema from rest_framework import filters, status -from rest_framework.decorators import action, api_view, permission_classes +from rest_framework.decorators import action from rest_framework.exceptions import ValidationError from rest_framework.mixins import ( CreateModelMixin, @@ -122,7 +122,8 @@ def speech_to_text(self, *args, **kwarg): stt_engine = Project.objects.get(external_id=project_id).stt_engine except Project.DoesNotExist: return Response( - {"error": "Project not found"}, status=status.HTTP_400_BAD_REQUEST + {"error": "Project not found"}, + status=status.HTTP_400_BAD_REQUEST, ) try: stats["transcript_start_time"] = time.time() @@ -139,7 +140,8 @@ def speech_to_text(self, *args, **kwarg): ) return Response( - {"transcript": translated_text, "stats": stats}, status=status.HTTP_200_OK + {"transcript": translated_text, "stats": stats}, + status=status.HTTP_200_OK, ) @extend_schema( diff --git a/ayushma/views/orphan.py b/ayushma/views/orphan.py index 046ba1fd..0d94abb6 100644 --- a/ayushma/views/orphan.py +++ b/ayushma/views/orphan.py @@ -35,11 +35,7 @@ class APIKeyAuth(permissions.BasePermission): def has_permission(self, request, view): if request.headers.get("X-API-KEY"): api_key = request.headers.get("X-API-KEY") - try: - APIKey.objects.get(key=api_key) - return True - except APIKey.DoesNotExist: - return False + return APIKey.objects.filter(key=api_key).exists() class OrphanChatViewSet( diff --git a/ayushma/views/service.py b/ayushma/views/service.py index 04a7b8a9..d9a3396a 100644 --- a/ayushma/views/service.py +++ b/ayushma/views/service.py @@ -1,28 +1,10 @@ -import time -from types import SimpleNamespace - -import openai -from django.conf import settings -from django.http import StreamingHttpResponse -from drf_spectacular.utils import extend_schema, extend_schema_view, inline_serializer -from rest_framework import permissions, status -from rest_framework.decorators import action -from rest_framework.exceptions import ValidationError +from rest_framework import permissions from rest_framework.mixins import CreateModelMixin -from rest_framework.parsers import MultiPartParser -from rest_framework.response import Response -from rest_framework.serializers import CharField, IntegerField -from ayushma.models import APIKey, Chat, ChatMessage, Project -from ayushma.serializers import ChatDetailSerializer, ConverseSerializer +from ayushma.models import APIKey from ayushma.serializers.services import TempTokenSerializer -from ayushma.utils.converse import converse_api -from ayushma.utils.language_helpers import translate_text -from ayushma.utils.openaiapi import converse from utils.views.base import BaseModelViewSet -from .chat import ChatViewSet - class Struct: def __init__(self, **entries): @@ -33,11 +15,7 @@ class APIKeyAuth(permissions.BasePermission): def has_permission(self, request, view): if request.headers.get("X-API-KEY"): api_key = request.headers.get("X-API-KEY") - try: - key = APIKey.objects.get(key=api_key) - return True - except APIKey.DoesNotExist: - return False + return APIKey.objects.filter(key=api_key).exists() class TempTokenViewSet(BaseModelViewSet, CreateModelMixin): diff --git a/ayushma/views/token.py b/ayushma/views/token.py index 7857f5fc..8af0e901 100644 --- a/ayushma/views/token.py +++ b/ayushma/views/token.py @@ -148,7 +148,7 @@ def validation(password): # find the token try: - reset_password_token = ResetPasswordToken.objects.get(key=token) + ResetPasswordToken.objects.get(key=token) except ResetPasswordToken.DoesNotExist: return Response(status=status.HTTP_400_BAD_REQUEST) diff --git a/core/settings/local.py b/core/settings/local.py index 2320a6da..ea7710e0 100644 --- a/core/settings/local.py +++ b/core/settings/local.py @@ -35,9 +35,7 @@ # WhiteNoise # ------------------------------------------------------------------------------ # http://whitenoise.evans.io/en/latest/django.html#using-whitenoise-in-development -INSTALLED_APPS = [ - "whitenoise.runserver_nostatic" -] + INSTALLED_APPS # noqa F405 +INSTALLED_APPS = ["whitenoise.runserver_nostatic"] + INSTALLED_APPS # noqa F405 # django-debug-toolbar diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..52ea391e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,24 @@ +[tool.isort] +profile = "black" +known_third_party = [ + "bs4", + "celery", + "corsheaders", + "django", + "django_filters", + "djangoql", + "drf_spectacular", + "environ", + "google", + "langchain", + "nltk", + "openai", + "pinecone", + "pypdf2", + "requests", + "rest_framework", + "rest_framework_simplejw", + "sentry_sdk", + "simple_history", + "tiktoken" +] diff --git a/setup.cfg b/setup.cfg index 9dcdd6bb..c0b76cdb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,7 +27,3 @@ exclude = __pycache__, .venv, migrations, - - -[isort] -profile = black