From ba7a5708393c922e3407c373720836f8c3aba588 Mon Sep 17 00:00:00 2001 From: Ismael Cruz Date: Thu, 5 Dec 2024 15:17:52 -0500 Subject: [PATCH 1/9] update deploy tags and location of local docker.sock file --- deployment/deploy_azure.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/deployment/deploy_azure.py b/deployment/deploy_azure.py index 38fe6500..ee00a426 100644 --- a/deployment/deploy_azure.py +++ b/deployment/deploy_azure.py @@ -14,10 +14,15 @@ import docker from dotenv import load_dotenv +# client = docker.from_env() load_dotenv() container_registry = os.getenv("AZURE_CONTAINER_REGISTRY") repo = os.getenv("AZURE_CONTAINER_REGISTRY_REPO") +print('### DEBUG ###') +print(f'container_registry: {container_registry}') +print(f'repo: {repo}') +print(f'postgres data host: {os.getenv("POSTGRES_DATA_HOST")}') # Script is run from top directory docker_compose_file = "docker-compose-deploy.yml" @@ -25,8 +30,11 @@ if sys.platform == "darwin": print("Running on Mac") + print(f"container_registry: {container_registry}") + print(f"repo: {repo}") client = docker.DockerClient( - base_url="unix:///Users/matthewharris/.docker/run/docker.sock " + # base_url="unix:///Users/matthewharris/.docker/run/docker.sock " + base_url="unix:///Users/t.o./.docker/run/docker.sock " ) else: client = docker.from_env() @@ -72,8 +80,8 @@ def deploy(): sys.exit() tags = { - "data-recipes-ai-server": [f"{container_registry}/{repo}", "server"], - "data-recipes-ai-chat": [f"{container_registry}/{repo}", "chat"], + "data-recipes-ai-server:latest": [f"{container_registry}/{repo}", "server"], + "data-recipes-ai-chat:latest": [f"{container_registry}/{repo}", "chat"], } run_cmd("az login") @@ -87,13 +95,15 @@ def deploy(): f"DOCKER_DEFAULT_PLATFORM={azure_platform} && docker compose -f {docker_compose_file} build" ) + # run_cmd("docker compose build") + for image in tags.keys(): print(f"Tagging {image} image ... with tag {tags[image][0]}:{tags[image][1]}") client.images.get(image).tag(tags[image][0], tags[image][1]) print(f"Pushing {image} image ... to {tags[image][0]}:{tags[image][1]}") client.images.push(tags[image][0], tags[image][1]) - sys.exit() + # sys.exit() run_cmd(f"docker compose -f {docker_compose_file} down") run_cmd(f"docker compose -f {docker_compose_file} pull") From d0ed787b54af271b22c0081ccf48aa6c3682bf2f Mon Sep 17 00:00:00 2001 From: Ismael Cruz Date: Thu, 5 Dec 2024 15:18:47 -0500 Subject: [PATCH 2/9] use later version of pgvector --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8b11be6c..9e26e688 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,7 @@ langchain_openai==0.1.7 literalai==0.0.604 matplotlib==3.9.0 pandas==2.2.2 -pgvector==0.2.4 +pgvector==0.2.5 plotly==5.22.0 psycopg2_binary==2.9.9 pyarrow==16.1.0 From 655b74146d5ba52922e50056415720e3397a8765 Mon Sep 17 00:00:00 2001 From: Ismael Cruz Date: Thu, 5 Dec 2024 15:20:13 -0500 Subject: [PATCH 3/9] update order of dependencies in yaml --- server/robocorp/actions_plugins/recipe-server/package.yaml | 4 ++-- server/robocorp/package.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/robocorp/actions_plugins/recipe-server/package.yaml b/server/robocorp/actions_plugins/recipe-server/package.yaml index b1d4ce05..22421ee6 100644 --- a/server/robocorp/actions_plugins/recipe-server/package.yaml +++ b/server/robocorp/actions_plugins/recipe-server/package.yaml @@ -21,8 +21,7 @@ dependencies: - robocorp-actions=0.0.7 - langchain=0.1.16 - langchain_community=0.0.34 - - langchain_openai=0.1.3 - - psycopg2-binary=2.9.3 + - langchain_openai=0.1.3 - pgvector=0.2.5 - python-dotenv=0.19.1 - pillow=10.3.0 @@ -32,3 +31,4 @@ dependencies: - seaborn=0.13.2 - geopandas=0.10.2 - hdx_python_api=6.2.4 + - psycopg2-binary=2.9.3 diff --git a/server/robocorp/package.yaml b/server/robocorp/package.yaml index 873e976e..fcd38f2e 100644 --- a/server/robocorp/package.yaml +++ b/server/robocorp/package.yaml @@ -23,6 +23,6 @@ dependencies: - langchain=0.1.16 - langchain_community=0.0.34 - langchain_openai=0.1.3 - - psycopg2-binary=2.9.3 - pgvector=0.2.5 + - psycopg2-binary=2.9.3 From 61bdd1341ee89d4f0bb496589e80cc53c4bc1dc9 Mon Sep 17 00:00:00 2001 From: Ismael Cruz Date: Thu, 5 Dec 2024 15:21:24 -0500 Subject: [PATCH 4/9] prevent pgvector extension from being auto created --- utils/recipes.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/utils/recipes.py b/utils/recipes.py index 8978d208..e4eb284f 100644 --- a/utils/recipes.py +++ b/utils/recipes.py @@ -88,15 +88,21 @@ def initialize_vector_db(): db = {} embedding_model, chat = get_models() + print(f'### embedding model: {embedding_model} ###') + print(f'### chat: {chat} ###') # This will create store tables if they don't exist + print('### create store tables ###') for mem_type in similarity_cutoff.keys(): + print(f'### {mem_type} ###') COLLECTION_NAME = f"{mem_type}_embedding" db[mem_type] = PGVector( collection_name=COLLECTION_NAME, - connection_string=CONNECTION_STRING, + connection_string=CONNECTION_STRING,git embedding_function=embedding_model, + create_extension=False ) + print(f'### {mem_type} SUCCESS ###') return db @@ -176,6 +182,7 @@ def check_recipe_memory(intent, debug=True): if debug: print(f"======= Checking {mem_type} for intent: {intent}") docs = db[mem_type].similarity_search_with_score(intent, k=3) + print('### docs SUCCESS ###') for d in docs: score = d[1] content = d[0].page_content From f9e6c4e19a7f2f1a098c45a0831009bca4110db0 Mon Sep 17 00:00:00 2001 From: Ismael Cruz Date: Thu, 5 Dec 2024 15:48:23 -0500 Subject: [PATCH 5/9] fix typos and remove comments --- deployment/deploy_azure.py | 4 ---- utils/recipes.py | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/deployment/deploy_azure.py b/deployment/deploy_azure.py index ee00a426..3942a6cb 100644 --- a/deployment/deploy_azure.py +++ b/deployment/deploy_azure.py @@ -19,10 +19,6 @@ container_registry = os.getenv("AZURE_CONTAINER_REGISTRY") repo = os.getenv("AZURE_CONTAINER_REGISTRY_REPO") -print('### DEBUG ###') -print(f'container_registry: {container_registry}') -print(f'repo: {repo}') -print(f'postgres data host: {os.getenv("POSTGRES_DATA_HOST")}') # Script is run from top directory docker_compose_file = "docker-compose-deploy.yml" diff --git a/utils/recipes.py b/utils/recipes.py index e4eb284f..0c947e49 100644 --- a/utils/recipes.py +++ b/utils/recipes.py @@ -98,9 +98,9 @@ def initialize_vector_db(): COLLECTION_NAME = f"{mem_type}_embedding" db[mem_type] = PGVector( collection_name=COLLECTION_NAME, - connection_string=CONNECTION_STRING,git + connection_string=CONNECTION_STRING, embedding_function=embedding_model, - create_extension=False + create_extension=False, ) print(f'### {mem_type} SUCCESS ###') From 563727899ff0f5e1eef3f662c5a7da2ca432843d Mon Sep 17 00:00:00 2001 From: Ismael Cruz Date: Thu, 5 Dec 2024 15:53:45 -0500 Subject: [PATCH 6/9] remove debugging comments --- utils/recipes.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/utils/recipes.py b/utils/recipes.py index 0c947e49..46de730b 100644 --- a/utils/recipes.py +++ b/utils/recipes.py @@ -88,13 +88,9 @@ def initialize_vector_db(): db = {} embedding_model, chat = get_models() - print(f'### embedding model: {embedding_model} ###') - print(f'### chat: {chat} ###') # This will create store tables if they don't exist - print('### create store tables ###') for mem_type in similarity_cutoff.keys(): - print(f'### {mem_type} ###') COLLECTION_NAME = f"{mem_type}_embedding" db[mem_type] = PGVector( collection_name=COLLECTION_NAME, @@ -102,7 +98,6 @@ def initialize_vector_db(): embedding_function=embedding_model, create_extension=False, ) - print(f'### {mem_type} SUCCESS ###') return db @@ -182,7 +177,6 @@ def check_recipe_memory(intent, debug=True): if debug: print(f"======= Checking {mem_type} for intent: {intent}") docs = db[mem_type].similarity_search_with_score(intent, k=3) - print('### docs SUCCESS ###') for d in docs: score = d[1] content = d[0].page_content From 9af528b5f7564546084fcaeab4cc9e9f84db253b Mon Sep 17 00:00:00 2001 From: Ismael Cruz Date: Thu, 5 Dec 2024 16:03:45 -0500 Subject: [PATCH 7/9] update docker compose command --- .github/workflows/e2e_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e_tests.yml b/.github/workflows/e2e_tests.yml index dc1a672c..4f805686 100644 --- a/.github/workflows/e2e_tests.yml +++ b/.github/workflows/e2e_tests.yml @@ -81,8 +81,8 @@ jobs: # TODO this should be enhanced to use a buildx bake to leverage layer caching for faster builds, or push to repo and simply have a pull for the run # TODO docker-compose files should be refactored to use scopes instead of different versions for each environment echo "Starting docker containers for dbs and server ..." - docker-compose pull - docker-compose up -d --build datadb recipedb server chat + docker compose pull + docker compose up -d --build datadb recipedb server chat # TODO: For some reason, maybe buildkit, in Github docker compose builds the image differently, and it doesn't work. Individual image build works. docker build --build-arg OPENAI_API_KEY=$ASSISTANTS_API_KEY \ From 37594a6c8c91d07ff86982130c76fe54322818c1 Mon Sep 17 00:00:00 2001 From: Ismael Cruz Date: Thu, 5 Dec 2024 16:27:33 -0500 Subject: [PATCH 8/9] revert to docker-compose command --- .github/workflows/e2e_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e_tests.yml b/.github/workflows/e2e_tests.yml index 4f805686..dc1a672c 100644 --- a/.github/workflows/e2e_tests.yml +++ b/.github/workflows/e2e_tests.yml @@ -81,8 +81,8 @@ jobs: # TODO this should be enhanced to use a buildx bake to leverage layer caching for faster builds, or push to repo and simply have a pull for the run # TODO docker-compose files should be refactored to use scopes instead of different versions for each environment echo "Starting docker containers for dbs and server ..." - docker compose pull - docker compose up -d --build datadb recipedb server chat + docker-compose pull + docker-compose up -d --build datadb recipedb server chat # TODO: For some reason, maybe buildkit, in Github docker compose builds the image differently, and it doesn't work. Individual image build works. docker build --build-arg OPENAI_API_KEY=$ASSISTANTS_API_KEY \ From d87c800b6a4fbf55dbe33361250065907932332d Mon Sep 17 00:00:00 2001 From: Ismael Cruz Date: Thu, 5 Dec 2024 16:31:26 -0500 Subject: [PATCH 9/9] use command docker compose --- .github/workflows/e2e_tests.yml | 4 ++-- .github/workflows/get_memory_test.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e_tests.yml b/.github/workflows/e2e_tests.yml index dc1a672c..4f805686 100644 --- a/.github/workflows/e2e_tests.yml +++ b/.github/workflows/e2e_tests.yml @@ -81,8 +81,8 @@ jobs: # TODO this should be enhanced to use a buildx bake to leverage layer caching for faster builds, or push to repo and simply have a pull for the run # TODO docker-compose files should be refactored to use scopes instead of different versions for each environment echo "Starting docker containers for dbs and server ..." - docker-compose pull - docker-compose up -d --build datadb recipedb server chat + docker compose pull + docker compose up -d --build datadb recipedb server chat # TODO: For some reason, maybe buildkit, in Github docker compose builds the image differently, and it doesn't work. Individual image build works. docker build --build-arg OPENAI_API_KEY=$ASSISTANTS_API_KEY \ diff --git a/.github/workflows/get_memory_test.yml b/.github/workflows/get_memory_test.yml index 25ae5e18..5f0b2856 100644 --- a/.github/workflows/get_memory_test.yml +++ b/.github/workflows/get_memory_test.yml @@ -77,8 +77,8 @@ jobs: ls data/datadb echo "Starting docker containers for dbs and server ..." - docker-compose pull - docker-compose up -d --build datadb recipedb server datadb + docker compose pull + docker compose up -d --build datadb recipedb server datadb sleep 10