Switched to Milvus

.envs/.local/.django

@@ -5,6 +5,7 @@ IPYTHONDIR=/app/.ipython
 # Redis
 # ------------------------------------------------------------------------------
 REDIS_URL=redis://redis:6379/0
+DATABASE_URL=postgres://debug:debug@postgres:5432/ayushma
 
 # Celery
 # ------------------------------------------------------------------------------

Makefile

@@ -30,7 +30,10 @@ logs:
 	docker compose -f $(docker_config_file) logs
 
 makemigrations: up
-	docker exec django bash -c "python manage.py makemigrations"
+	docker compose -f $(docker_config_file) exec django bash -c "python manage.py makemigrations"
+
+migrate: up
+	docker compose -f $(docker_config_file) exec django bash -c "python manage.py migrate"
 
 checkmigration:
 	docker compose -f $(docker_config_file) exec django bash -c "python manage.py makemigrations --check --dry-run"

Pipfile

@@ -24,7 +24,7 @@ drf-nested-routers = "==0.93.4"
 drf-spectacular = "==0.27.0"
 requests = "==2.31.0"
 openai = "==1.2.4"
-pinecone-client = "==2.2.4"
+pinecone-client = "==4.1.1"
 pypdf2 = "==3.0.1"
 tiktoken = "==0.5.2"
 langchain = "==0.0.352"
@@ -41,6 +41,7 @@ nltk = "==3.8.1"
 gunicorn = "==21.2.0"
 psycopg = {extras = ["c"], version = "==3.1.17"}
 sentry-sdk = "==1.30.0"
+pymilvus = "*"
 
 [dev-packages]
 ipdb = "==0.13.13"

README.md

@@ -1,50 +1,48 @@
 # Ayushma AI
 
-This is a Django Project.
+This is a Django Project
 
 ## Requirements
 
-- Python 3.8
-- Postgres 15
 - OpenAI Account with a valid API Key
 - Pinecone Account with a valid API Key
+- Docker
 
 ## Optional Requirements
 
-- Docker
 - AWS SES Accout
 - AWS S3 Account
 - Google Cloud account with access to speech to text API
 
-## Installation
+## Running the Project
 
-create a virtual environment and install the requirements
+> We strongly recommend using Docker to run the project. Manual methods are not supported at the moment.
 
 ```bash
-pip3 install --user virtualenv
-virtualenv .venv
-source .venv/bin/activate
-pip3 install -r requirements/local.txt
+make up
 ```
 
 ## Env Variables
 
-You can add these at the end of your `activate` file in `[virtualenvfolder] -> bin` like `export [env] = [value]`
-
-| Variable | Description
-| --- | ---
-| AI_NAME | Name of the AI (default: Ayushma)
-| OPENAI_API_KEY | OpenAI API Key
-| PINECONE_API_KEY | Pinecone API Key
-| PINECONE_ENVIRONMENT | Pinecone Environment
-| PINECONE_INDEX | Pinecone Index
-| CURRENT_DOMAIN | Current Domain where the frontend is hosted. ex. `https://ayushma.ohc.network`
-| EMAIL_HOST | SES Email Host (Optional)
-| EMAIL_USER | SES Email User (Optional)
-| EMAIL_PASSWORD | SES Email Password (Optional)
-| GOOGLE_APPLICATION_CREDENTIALS | Google Cloud Credentials (Optional). These should be in a file named `gc_credential.json` in the root of the project
-| S3_SECRET_KEY | AWS S3 Secret Key (Optional)
-| S3_KEY_ID | AWS S3 Key ID (Optional)
-| S3_BUCKET_NAME | AWS S3 Bucket Name (Optional)
-| S3_REGION | AWS S3 Region (Optional)
-| GOOGLE_RECAPTCHA_SECRET_KEY | Google Recaptcha Secret Key (Optional)
+Add these environment variables to your `.env` file.
+
+| Variable                       | Description                                                                                                          |
+| ------------------------------ | -------------------------------------------------------------------------------------------------------------------- |
+| AI_NAME                        | Name of the AI (default: Ayushma)                                                                                    |
+| OPENAI_API_KEY                 | OpenAI API Key                                                                                                       |
+| PINECONE_API_KEY               | Pinecone API Key                                                                                                     |
+| PINECONE_INDEX                 | Pinecone Index                                                                                                       |
+| CURRENT_DOMAIN                 | Current Domain where the frontend is hosted. ex. `https://ayushma.ohc.network`                                       |
+| EMAIL_HOST                     | SES Email Host (Optional)                                                                                            |
+| EMAIL_USER                     | SES Email User (Optional)                                                                                            |
+| EMAIL_PASSWORD                 | SES Email Password (Optional)                                                                                        |
+| GOOGLE_APPLICATION_CREDENTIALS | Google Cloud Credentials (Optional). These should be in a file named `gc_credential.json` in the root of the project |
+| S3_SECRET_KEY                  | AWS S3 Secret Key (Optional)                                                                                         |
+| S3_KEY_ID                      | AWS S3 Key ID (Optional)                                                                                             |
+| S3_BUCKET_NAME                 | AWS S3 Bucket Name (Optional)                                                                                        |
+| S3_REGION                      | AWS S3 Region (Optional)                                                                                             |
+| GOOGLE_RECAPTCHA_SECRET_KEY    | Google Recaptcha Secret Key (Optional)                                                                               |
+
+## Google Cloud
+
+To use Google Cloud Speech to Text API, you need to enable the API and create a service account. Download the credentials and save them in a file named `gc_credential.json` in the root of the project.

ayushma/migrations/0053_alter_chat_model_alter_project_model.py

@@ -0,0 +1,46 @@
+# Generated by Django 4.2.6 on 2024-07-02 16:07
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("ayushma", "0052_document_failed"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="chat",
+            name="model",
+            field=models.IntegerField(
+                blank=True,
+                choices=[
+                    (1, "Gpt 3 5"),
+                    (2, "Gpt 3 5 16K"),
+                    (3, "Gpt 4"),
+                    (4, "Gpt 4 32K"),
+                    (5, "Gpt 4 Visual"),
+                    (6, "Gpt 4 Turbo"),
+                    (7, "Gpt 4 Omni"),
+                ],
+                null=True,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="project",
+            name="model",
+            field=models.IntegerField(
+                choices=[
+                    (1, "Gpt 3 5"),
+                    (2, "Gpt 3 5 16K"),
+                    (3, "Gpt 4"),
+                    (4, "Gpt 4 32K"),
+                    (5, "Gpt 4 Visual"),
+                    (6, "Gpt 4 Turbo"),
+                    (7, "Gpt 4 Omni"),
+                ],
+                default=1,
+            ),
+        ),
+    ]

ayushma/models/enums.py

@@ -47,6 +47,7 @@ class ModelType(IntegerChoices):
     GPT_4_32K = 4
     GPT_4_VISUAL = 5
     GPT_4_TURBO = 6
+    GPT_4_OMNI = 7
 
 
 class StatusChoices(IntegerChoices):

ayushma/utils/langchain.py

@@ -2,7 +2,8 @@
 
 import openai
 from django.conf import settings
-from langchain import LLMChain, PromptTemplate
+from langchain.chains import LLMChain
+from langchain.prompts import PromptTemplate
 from langchain.callbacks.manager import AsyncCallbackManager
 from langchain.chat_models import ChatOpenAI
 from langchain.llms import AzureOpenAI
@@ -38,6 +39,8 @@ def get_model_name(model_type: ModelType):
         return "gpt-4-vision-preview"
     elif model_type == ModelType.GPT_4_TURBO:
         return "gpt-4-1106-preview"
+    elif model_type == ModelType.GPT_4_OMNI:
+        return "gpt-4o"
     else:
         if settings.OPENAI_API_TYPE == "azure":
             return settings.AZURE_CHAT_MODEL

ayushma/utils/openaiapi.py

@@ -10,14 +10,14 @@
 from django.conf import settings
 from langchain.schema import AIMessage, HumanMessage
 from openai import OpenAI
-from pinecone import QueryResponse
 
 from ayushma.models import ChatMessage
 from ayushma.models.chat import Chat
 from ayushma.models.document import Document
 from ayushma.models.enums import ChatMessageType, ModelType
 from ayushma.utils.langchain import LangChainHelper
 from ayushma.utils.language_helpers import text_to_speech, translate_text
+from ayushma.utils.vectordb import VectorDB
 from core.settings.base import AI_NAME
 
 
@@ -67,39 +67,6 @@ def get_embedding(
     return [record.embedding for record in res.data]
 
 
-def get_sanitized_reference(pinecone_references: List[QueryResponse]) -> str:
-    """
-    Extracts the text from the Pinecone QueryResponse object and sanitizes it.
-
-    Args:
-        pinecone_reference (List[QueryResponse]): The similar documents retrieved from
-            the Pinecone index.
-
-    Returns:
-        A string containing the document id and text from the Pinecone QueryResponse object.
-
-    Example usage:
-        >>> get_sanitized_reference([QueryResponse(...), QueryResponse(...)])
-        "{'28': 'Hello how are you, I am fine, thank you.', '21': 'How was your day?, Mine was good.'}"
-    """
-    sanitized_reference = {}
-
-    for reference in pinecone_references:
-        for match in reference.matches:
-            try:
-                document_id = str(match.metadata["document"])
-                text = str(match.metadata["text"]).replace("\n", " ") + ","
-                if document_id in sanitized_reference:
-                    sanitized_reference[document_id] += text
-                else:
-                    sanitized_reference[document_id] = text
-            except Exception as e:
-                print(f"Error extracting reference: {e}")
-                pass
-
-    return json.dumps(sanitized_reference)
-
-
 def num_tokens_from_string(string: str, encoding_name: str) -> int:
     """Returns the number of tokens in a text string."""
     encoding = tiktoken.get_encoding(encoding_name)
@@ -155,18 +122,18 @@ def get_reference(text, openai_key, namespace, top_k):
                 raise Exception(
                     "[Reference] Error generating embeddings for split text"
                 )
-    # find similar embeddings from pinecone index for each embedding
-    pinecone_references: List[QueryResponse] = []
-
-    for embedding in embeddings:
-        similar: QueryResponse = settings.PINECONE_INDEX_INSTANCE.query(
-            vector=embedding,
-            top_k=int(top_k),
-            namespace=namespace,
-            include_metadata=True,
-        )
-        pinecone_references.append(similar)
-    return get_sanitized_reference(pinecone_references=pinecone_references)
+    # find similar embeddings from vector index for each embedding    
+
+    flat_embeddings = [item for sublist in embeddings for item in sublist]
+    vdb = VectorDB()
+    similar = vdb.search(
+        embeddings=flat_embeddings,
+        partition_name=namespace,
+        limit=int(top_k),
+    )
+
+    print("References fetched")
+    return vdb.sanitize(similar)
 
 
 def add_reference_documents(chat_message):
@@ -299,7 +266,7 @@ def converse(
             reference = get_reference(
                 english_text,
                 openai_key,
-                str(chat.project.external_id),
+                str(chat.project.external_id).replace("-", "_"),
                 match_number,
             )
         except Exception as e: