Merging main into feature branch to get latest changes

.env.example

@@ -23,6 +23,16 @@ DOMAIN_SERVER=http://localhost:3080
 
 NO_INDEX=true
 
+ Connection string for where the data DB resides. Can be remove or local DB (Remote)
+#POSTGRES_DATA_HOST=postgres-prototypes.postgres.database.azure.com
+#POSTGRES_DATA_PORT=5432
+#POSTGRES_DATA_DB=hdexpert-alpha
+#POSTGRES_DATA_USER=bots_rule3
+#POSTGRES_DATA_PASSWORD=96__Mou9!2Yl
+
+# Only used in deployment
+REMOTE_DB_CONN_STRING=postgresql://<LOGIN GOES HERE>:<PASWORD HERE>@<HOST>:<PORT>/<DATABASE>
+
 # =========== Local Docker containers START ==========
 POSTGRES_DATA_HOST=postgres-prototypes.postgres.database.azure.com
 POSTGRES_DATA_PORT=5432
@@ -39,15 +49,18 @@ POSTGRES_RECIPE_PASSWORD=
 RECIPE_DB_CONN_STRING=postgresql://${POSTGRES_RECIPE_USER}:${POSTGRES_RECIPE_PASSWORD}@${POSTGRES_RECIPE_HOST}:${POSTGRES_RECIPE_PORT}/${POSTGRES_RECIPE_DB}
 # =========== Local Docker containers END ==========
 
+CONFIG_PATH="/app/librechat.yaml"
+
 #==================================================#
 #               Get Memory Action Configuration    #
 #==================================================#
-OPENAI_API_TYPE=azure
-OPENAI_API_ENDPOINT=
-OPENAI_API_VERSION_MEMORY=2024-02-15-preview
-BASE_URL_MEMORY=
-MODEL_MEMORY=gpt-4-turbo
-OPENAI_TEXT_COMPLETION_DEPLOYMENT_NAME=text-embedding-ada-002
+RECIPES_OPENAI_API_TYPE=azure
+RECIPES_OPENAI_API_KEY=
+RECIPES_OPENAI_API_ENDPOINT=
+RECIPES_OPENAI_API_VERSION=2024-02-15-preview
+RECIPES_BASE_URL=
+RECIPES_MODEL=gpt-4-turbo
+RECIPES_OPENAI_TEXT_COMPLETION_DEPLOYMENT_NAME=text-embedding-ada-002
 
 #===============#
 # JSON Logging  #
@@ -111,7 +124,7 @@ ANTHROPIC_API_KEY=sk-ant-api03-6b0Vg33VrXpxmVALpTlGJu90IvfrIbW5Q8wEamdnk1Es4mg6e
 # Azure      #
 #============#
 
-AZURE_API_KEY=
+AZURE_API_KEY_ENV=
 
 # Note: these variables are DEPRECATED
 # Use the `librechat.yaml` configuration for `azureOpenAI` instead
@@ -382,7 +395,7 @@ FIREBASE_APP_ID=
 
 APP_TITLE=LibreChat
 # CUSTOM_FOOTER="My custom footer"
-HELP_AND_FAQ_URL=https://librechat.ai
+HELP_AND_FAQ_URL=https://www.datakind.org/privacy-policy/
 
 # SHOW_BIRTHDAY_ICON=true
 

.gitignore

@@ -9,4 +9,9 @@ tmp
 ingestion/api/hapi/*
 ingestion/api/hdx/*
 .vscode
-code-interpretor/static/*
+code-interpretor/static/*
+actions/actions_plugins/recipe-server/images
+__pycache__
+ui/recipes_assistant_chat/datadb
+ui/recipes_assistant_chat/docsdb
+ui/recipes_assistant_chat/recipesdb

README.md

@@ -33,6 +33,11 @@ Robocorp AI Actions API - [http://localhost:3001/](http://localhost:3001/)
 TODO: This will be automated, but for now ...
 
 1. Got to  [chat app](http://localhost:3080/) and register a user on the login page
+2. Select "Plugins' endpoint at top, then in the plugin box go to store and activate 
+   - Humanitariuan Data Assistant
+   - Humanitarian Data Recipes
+   - Code sherpa, when asked enter URL http://code-interpretor:3333
+3. Populate system prompts, see `./assistant/recipes_assistant/prompts`, build presets
 2. Log in
 3. `docker exec -it haa-ingestion /bin/bash`
 4. `python3 ingest.py`
@@ -71,11 +76,12 @@ One thing to mention on an Azure deploy, it that doesn't get pushed to the web a
 
 ### On a Mac
 
-Make dure docker has 'Use Rosetta for x86_64/amd64 emulation on Apple Silicon' set in settings if using a MAC silicon chip. The deploy script can then build images that wwok on Azure then revert to images that work on your Mac.
+Make sure docker has 'Use Rosetta for x86_64/amd64 emulation on Apple Silicon' set in settings if using a MAC silicon chip. The deploy script can then build images that wwok on Azure then 
+revert to images that work on your Mac.
 
 Note: 
 
-`docker-compose-azure.yml` is the configurtation used in the deployment center screen on the web app
+`docker-compose-azure.yml` is the configutation used in the deployment center screen on the web app
 `docker-compose.yml` is used for building locally
 
 ## Databases

actions/Dockerfile

@@ -27,15 +27,18 @@ ADD https://downloads.robocorp.com/action-server/releases/latest/linux64/action-
 RUN chmod +x /usr/local/bin/action-server
 
 COPY ./actions .
+
+RUN mkdir -p /action-server/actions/actions_plugins/recipe-server/images
+
 RUN chown -R as-user:as-user /action-server
 
+RUN chown -R as-user:as-user /action-server/actions/actions_plugins/recipe-server/images
+
 USER as-user
 
 ARG DATA_DB_CONN_STRING
 ENV DATA_DB_CONN_STRING=$DATA_DB_CONN_STRING
 
-# Load all actions
-# RUN action-server import --datadir=/action-server/datadir
 # Load individually
 RUN action-server import --dir=./actions_plugins/postgres-universal --datadir=/action-server/datadir
 RUN action-server import --dir=./actions_plugins/recipe-server --datadir=/action-server/datadir

actions/actions_plugins/postgres-universal/actions.py

@@ -5,7 +5,7 @@
 
 CONNECTION_FILE_PATH = "postgres_data_connection.json"
 
-MAX_CHARS_TOT = 10000
+MAX_CHARS_TOT = 100000
 
 
 class ReadOnlyConnection:

actions/actions_plugins/recipe-server/actions.py

@@ -97,13 +97,14 @@
 }
 
 conn_params = {
-    "OPENAI_API_TYPE": os.getenv("OPENAI_API_TYPE"),
-    "OPENAI_API_ENDPOINT": os.getenv("OPENAI_API_ENDPOINT"),
-    "OPENAI_API_VERSION": os.getenv("OPENAI_API_VERSION_MEMORY"),
-    "BASE_URL": os.getenv("BASE_URL_MEMORY"),
-    "MODEL": os.getenv("MODEL"),
-    "OPENAI_TEXT_COMPLETION_DEPLOYMENT_NAME": os.getenv(
-        "OPENAI_TEXT_COMPLETION_DEPLOYMENT_NAME"
+    "RECIPES_OPENAI_API_TYPE": os.getenv("RECIPES_OPENAI_API_TYPE"),
+    "RECIPES_OPENAI_API_KEY": os.getenv("RECIPES_OPENAI_API_KEY"),
+    "RECIPES_OPENAI_API_ENDPOINT": os.getenv("RECIPES_OPENAI_API_ENDPOINT"),
+    "RECIPES_OPENAI_API_VERSION": os.getenv("RECIPES_OPENAI_API_VERSION"),
+    "RECIPES_BASE_URL": os.getenv("RECIPES_BASE_URL"),
+    "RECIPES_MODEL": os.getenv("RECIPES_MODEL"),
+    "RECIPES_OPENAI_TEXT_COMPLETION_DEPLOYMENT_NAME": os.getenv(
+        "RECIPES_OPENAI_TEXT_COMPLETION_DEPLOYMENT_NAME"
     ),
     "POSTGRES_DB": os.getenv("POSTGRES_RECIPE_DB"),
     "POSTGRES_USER": os.getenv("POSTGRES_RECIPE_USER"),
@@ -205,11 +206,11 @@ def get_models(conn_params):
     Returns:
         _type_: _description_
     """
-    api_key = conn_params["OPENAI_API_KEY"]
-    base_url = conn_params["BASE_URL"]
-    api_version = conn_params["OPENAI_API_VERSION"]
-    api_type = conn_params["OPENAI_API_TYPE"]
-    completion_model = conn_params["OPENAI_TEXT_COMPLETION_DEPLOYMENT_NAME"]
+    api_key = conn_params["RECIPES_OPENAI_API_KEY"]
+    base_url = conn_params["RECIPES_BASE_URL"]
+    api_version = conn_params["RECIPES_OPENAI_API_VERSION"]
+    api_type = conn_params["RECIPES_OPENAI_API_TYPE"]
+    completion_model = conn_params["RECIPES_OPENAI_TEXT_COMPLETION_DEPLOYMENT_NAME"]
 
     if api_type == "openai":
         print("Using OpenAI API in memory.py")
@@ -435,7 +436,6 @@ def get_memory(user_input, chat_history, generate_intent=True) -> str:
     Performs a search in the memory for a given intent and returns the best match found.
 
     Args:
-        conn_params (str): The connection parameters for the database.
         user_input (str): The user input to search for in the memory.
         chat_history (str): The chat history.
         generate_intent (str): A flag to indicate whether to generate the intent from the chat history.

cleanup.sh

@@ -4,4 +4,4 @@ rm -rf ./ui/recipes_assistant_chat/images/*
 rm -rf ./ui/recipes_assistant_chat/logs/*
 rm -rf ./ui/recipes_assistant_chat/data-node/*
 rm -rf ./ui/recipes_assistant_chat/meili_data_v1.7/*
-rm -rf ./ui/recipes_assistant_chat/pgdata2/*
+rm -rf ./ui/recipes_assistant_chat/datadb/*

code-interpretor/Dockerfile

@@ -1,6 +1,16 @@
 # Base Ubuntu Image
 FROM ubuntu:22.04
 
+# Needed for geopandas poetry installation
+RUN apt-get update && apt-get install -y python3-dev
+RUN apt-get update && apt-get install -y --no-install-recommends \
+     libgeos-dev \
+     && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+RUN  apt-get update && apt-get install -y --no-install-recommends \
+     libgdal-dev \
+     && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+ENV GDAL_CONFIG=/usr/bin/gdal-config
+
 # Update and install necessary compilers/interpreters
 RUN apt-get update && apt-get install --no-install-recommends -y \
     python3.10 \
@@ -23,7 +33,8 @@ WORKDIR /app
 RUN git clone https://github.com/iamgreggarcia/codesherpa.git
 RUN mv codesherpa/* /app/
 
-#COPY ./pyproject.toml ./poetry.lock* /app/
+COPY ./code-interpretor/pyproject.toml /app/
+RUN rm ./poetry.lock
 
 RUN pip install --no-cache-dir poetry
 
@@ -32,11 +43,9 @@ RUN poetry config virtualenvs.in-project true && \
     poetry install --no-dev --no-root
 
 RUN /app/.venv/bin/pip install --no-cache-dir pytest pytest-cov pytest-asyncio
-
 RUN /app/.venv/bin/pip install --no-cache-dir google-cloud-secret-manager 
 
 RUN mkdir -p /app/static/images
-
 RUN mkdir -p /app/static/uploads
 
 ENV PYTHONPATH=/app
@@ -45,4 +54,5 @@ ENV PATH="/app/.venv/bin:${PATH}"
 
 #COPY . /app/
 
-CMD ["python", "-c", "import server.main; server.main.start()"]
+#CMD ["python", "-c", "import server.main; server.main.start()"]
+CMD ["python", "-c", "import localserver.main; localserver.main.start()"]

code-interpretor/pyproject.toml

@@ -0,0 +1,57 @@
+[tool.poetry]
+name = "codesherpa"
+version = "0.1.0"
+description = "Codesherpa is a code-interpreter API that allows you to run code in a Docker container."
+authors = ["Greg Garcia <[email protected]>"]
+license = "MIT"
+readme = "README.md"
+packages = [
+    { include = "localserver", from = "." }
+]
+
+
+[tool.poetry.dependencies]
+python = "^3.10, <3.12"
+fastapi = "^0.95.1"
+uvicorn = "^0.22.0"
+pydantic = "^1.10.7"
+matplotlib = "^3.7.1"
+seaborn = "^0.12.2"
+pandas = "^2.0.1"
+passlib = "^1.7.4"
+python-jose = "^3.3.0"
+slowapi = "^0.1.8"
+loguru = "^0.7.0"
+google-auth = "^2.18.0"
+httpx = "^0.24.0" 
+requests = "^2.30.0"
+jinja2 = "^3.1.2"
+python-multipart = "^0.0.6"
+numpy = "^1.24.3"
+scipy = "^1.10.1"
+openpyxl = "^3.1.2"
+scikit-learn = "^1.2.2"
+geoalchemy2 = "^0.14.6"
+geopandas = "^0.10.2" 
+psycopg2_binary = "^2.9.9"
+
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^7.2.1"
+pytest-cov = "^4.0.0"
+pytest-asyncio = "^0.20.3"
+loguru = "^0.7.0"
+flake8 = "^4.0.1"
+google-auth = "^2.18.0"
+requests = "^2.30.0"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.poetry.scripts]
+start = "localserver.main:start"
+dev = "localserver.main:dev"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

deploy_azure.py

@@ -18,15 +18,20 @@
 client = docker.DockerClient(base_url='unix:///Users/matthewharris/.docker/run/docker.sock ')
 
 container_registry = "dkdsprototypesreg01.azurecr.io"
+repo = "humanitarian-ai-assistant"
+
+# Note that the DB images are all changed to be DBs in Azure.
 tags = {
-    "ankane/pgvector:latest":     [f"{container_registry}/containergroup","vectordb"],
-    "getmeili/meilisearch:v1.7.3":  [f"{container_registry}/containergroup","meilisearch"],
-    "mongo":      [f"{container_registry}/containergroup","mongodb"],
-    "humanitarian_ai_assistant-actions":      [f"{container_registry}/containergroup","actions"],
-    "ghcr.io/danny-avila/librechat-rag-api-dev-lite:latest":      [f"{container_registry}/containergroup","rag_api"],
-    "humanitarian_ai_assistant-api":              [f"{container_registry}/containergroup","api"],
+    "humanitarian_ai_assistant-api":              [f"{container_registry}/{repo}","api"],
+    "getmeili/meilisearch:v1.7.3":                [f"{container_registry}/{repo}","meilisearch"],
+    "ghcr.io/danny-avila/librechat-rag-api-dev-lite:latest":      [f"{container_registry}/{repo}","rag_api"],
+    "ankane/pgvector:latest":                     [f"{container_registry}/{repo}","docsdb"],
+    "humanitarian_ai_assistant-actions":          [f"{container_registry}/{repo}","actions"],
+    "busybox":                                    [f"{container_registry}/{repo}","init"],
+    "nginx:latest":                               [f"{container_registry}/{repo}","nginx"],
+    "humanitarian_ai_assistant-code-interpretor":    [f"{container_registry}/{repo}","code-interpreter"],
 }
-docker_compose_file = "docker-compose.yml"
+docker_compose_file = "docker-compose-deploy.yml"
 azure_platform = "linux/amd64"
 
 def run_cmd(cmd):
@@ -54,5 +59,5 @@ def run_cmd(cmd):
 run_cmd(f"docker compose -f {docker_compose_file} build")
 run_cmd(f"docker compose -f {docker_compose_file} up -d")
 
-print("Done")
+print("Now go and click on https://ai-assistants-prototypes.azurewebsites.net/c/new to trigger to deploy")