diff --git a/building-agentic-rag-with-Llamaindex/README.md b/building-agentic-rag-with-Llamaindex/README.md new file mode 100644 index 0000000..15b2179 --- /dev/null +++ b/building-agentic-rag-with-Llamaindex/README.md @@ -0,0 +1,7 @@ +# Building Agentic RAG with Llamaindex + +![Building Agentic RAG with Llamaindex](summary-index.png) + +![Full Agent Reasioning](full-agent-reasoning-loop.png) + +![Three Documents-agent](three-documents-agent.png) diff --git a/building-agentic-rag-with-Llamaindex/full-agent-reasoning-loop.png b/building-agentic-rag-with-Llamaindex/full-agent-reasoning-loop.png new file mode 100644 index 0000000..4b199ac Binary files /dev/null and b/building-agentic-rag-with-Llamaindex/full-agent-reasoning-loop.png differ diff --git a/building-agentic-rag-with-Llamaindex/l2-tool-calling/L2_Tool_Calling.ipynb b/building-agentic-rag-with-Llamaindex/l2-tool-calling/L2_Tool_Calling.ipynb new file mode 100644 index 0000000..4f94b37 --- /dev/null +++ b/building-agentic-rag-with-Llamaindex/l2-tool-calling/L2_Tool_Calling.ipynb @@ -0,0 +1,435 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "46f32ff1", + "metadata": {}, + "source": [ + "# Lesson 2: Tool Calling" + ] + }, + { + "cell_type": "markdown", + "id": "fb345ad0", + "metadata": {}, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c5bbf530-3f05-434c-a70f-ac2cc4b8f7aa", + "metadata": { + "height": 47, + "tags": [] + }, + "outputs": [], + "source": [ + "from helper import get_openai_api_key\n", + "OPENAI_API_KEY = get_openai_api_key()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d4c06c95-e8b2-4574-b14d-685876aa1c47", + "metadata": { + "height": 47, + "tags": [] + }, + "outputs": [], + "source": [ + "import nest_asyncio\n", + "nest_asyncio.apply()" + ] + }, + { + "cell_type": "markdown", + "id": "2e53c064", + "metadata": {}, + "source": [ + "## 1. Define a Simple Tool" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "071b717a-93cc-4332-b357-59a693359563", + "metadata": { + "height": 234, + "tags": [] + }, + "outputs": [], + "source": [ + "from llama_index.core.tools import FunctionTool\n", + "\n", + "def add(x: int, y: int) -> int:\n", + " \"\"\"Adds two integers together.\"\"\"\n", + " return x + y\n", + "\n", + "def mystery(x: int, y: int) -> int: \n", + " \"\"\"Mystery function that operates on top of two numbers.\"\"\"\n", + " return (x + y) * (x + y)\n", + "\n", + "\n", + "add_tool = FunctionTool.from_defaults(fn=add)\n", + "mystery_tool = FunctionTool.from_defaults(fn=mystery)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d4e62118-992b-4629-9022-be8c628209c1", + "metadata": { + "height": 166, + "tags": [] + }, + "outputs": [], + "source": [ + "from llama_index.llms.openai import OpenAI\n", + "\n", + "llm = OpenAI(model=\"gpt-3.5-turbo\")\n", + "response = llm.predict_and_call(\n", + " [add_tool, mystery_tool], \n", + " \"Tell me the output of the mystery function on 2 and 9\", \n", + " verbose=True\n", + ")\n", + "print(str(response))" + ] + }, + { + "cell_type": "markdown", + "id": "8cb8a835", + "metadata": {}, + "source": [ + "## 2. Define an Auto-Retrieval Tool" + ] + }, + { + "cell_type": "markdown", + "id": "6589123f", + "metadata": {}, + "source": [ + "### Load Data" + ] + }, + { + "cell_type": "markdown", + "id": "bcdea238", + "metadata": { + "tags": [] + }, + "source": [ + "To download this paper, below is the needed code:\n", + "\n", + "#!wget \"https://openreview.net/pdf?id=VtmBAGCN7o\" -O metagpt.pdf\n", + "\n", + "**Note**: The pdf file is included with this lesson. To access it, go to the `File` menu and select`Open...`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fbe9326c-d7b3-452b-ae52-12f000157be4", + "metadata": { + "height": 64, + "tags": [] + }, + "outputs": [], + "source": [ + "from llama_index.core import SimpleDirectoryReader\n", + "# load documents\n", + "documents = SimpleDirectoryReader(input_files=[\"metagpt.pdf\"]).load_data()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5451f0a3-d0a6-4b5c-a337-8e1a343ff5f0", + "metadata": { + "height": 64, + "tags": [] + }, + "outputs": [], + "source": [ + "from llama_index.core.node_parser import SentenceSplitter\n", + "splitter = SentenceSplitter(chunk_size=1024)\n", + "nodes = splitter.get_nodes_from_documents(documents)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e0fe0a9c-1f87-4ae7-a79e-7c3cf9c395ed", + "metadata": { + "height": 30, + "tags": [] + }, + "outputs": [], + "source": [ + "print(nodes[0].get_content(metadata_mode=\"all\"))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d7965cba-67b8-4cca-8e5f-2b0dbc96f6b0", + "metadata": { + "height": 81, + "tags": [] + }, + "outputs": [], + "source": [ + "from llama_index.core import VectorStoreIndex\n", + "\n", + "vector_index = VectorStoreIndex(nodes)\n", + "query_engine = vector_index.as_query_engine(similarity_top_k=2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "560f319c-8479-40c5-9b55-480fef98deb7", + "metadata": { + "height": 251, + "tags": [] + }, + "outputs": [], + "source": [ + "from llama_index.core.vector_stores import MetadataFilters\n", + "\n", + "query_engine = vector_index.as_query_engine(\n", + " similarity_top_k=2,\n", + " filters=MetadataFilters.from_dicts(\n", + " [\n", + " {\"key\": \"page_label\", \"value\": \"2\"}\n", + " ]\n", + " )\n", + ")\n", + "\n", + "response = query_engine.query(\n", + " \"What are some high-level results of MetaGPT?\", \n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2da4042f-8fdb-4959-8760-86685c903cfd", + "metadata": { + "height": 30, + "tags": [] + }, + "outputs": [], + "source": [ + "print(str(response))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "30bb264c-42e0-46f8-9d28-da11a8535960", + "metadata": { + "height": 47, + "tags": [] + }, + "outputs": [], + "source": [ + "for n in response.source_nodes:\n", + " print(n.metadata)" + ] + }, + { + "cell_type": "markdown", + "id": "6c392482", + "metadata": {}, + "source": [ + "### Define the Auto-Retrieval Tool" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2639e79b-f615-425b-85ea-8a279bb26dd0", + "metadata": { + "height": 608, + "tags": [] + }, + "outputs": [], + "source": [ + "from typing import List\n", + "from llama_index.core.vector_stores import FilterCondition\n", + "\n", + "\n", + "def vector_query(\n", + " query: str, \n", + " page_numbers: List[str]\n", + ") -> str:\n", + " \"\"\"Perform a vector search over an index.\n", + " \n", + " query (str): the string query to be embedded.\n", + " page_numbers (List[str]): Filter by set of pages. Leave BLANK if we want to perform a vector search\n", + " over all pages. Otherwise, filter by the set of specified pages.\n", + " \n", + " \"\"\"\n", + "\n", + " metadata_dicts = [\n", + " {\"key\": \"page_label\", \"value\": p} for p in page_numbers\n", + " ]\n", + " \n", + " query_engine = vector_index.as_query_engine(\n", + " similarity_top_k=2,\n", + " filters=MetadataFilters.from_dicts(\n", + " metadata_dicts,\n", + " condition=FilterCondition.OR\n", + " )\n", + " )\n", + " response = query_engine.query(query)\n", + " return response\n", + " \n", + "\n", + "vector_query_tool = FunctionTool.from_defaults(\n", + " name=\"vector_tool\",\n", + " fn=vector_query\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2a408ace-cf25-425b-8248-7028ceabcd42", + "metadata": { + "height": 115, + "tags": [] + }, + "outputs": [], + "source": [ + "llm = OpenAI(model=\"gpt-3.5-turbo\", temperature=0)\n", + "response = llm.predict_and_call(\n", + " [vector_query_tool], \n", + " \"What are the high-level results of MetaGPT as described on page 2?\", \n", + " verbose=True\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6ec05565-6adf-4294-ba5c-b384220876ac", + "metadata": { + "height": 47, + "tags": [] + }, + "outputs": [], + "source": [ + "for n in response.source_nodes:\n", + " print(n.metadata)" + ] + }, + { + "cell_type": "markdown", + "id": "fef4dec0", + "metadata": {}, + "source": [ + "## Let's add some other tools!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "55dd32e5-e29f-42ed-839a-ca937fe4743e", + "metadata": { + "height": 268, + "tags": [] + }, + "outputs": [], + "source": [ + "from llama_index.core import SummaryIndex\n", + "from llama_index.core.tools import QueryEngineTool\n", + "\n", + "summary_index = SummaryIndex(nodes)\n", + "summary_query_engine = summary_index.as_query_engine(\n", + " response_mode=\"tree_summarize\",\n", + " use_async=True,\n", + ")\n", + "summary_tool = QueryEngineTool.from_defaults(\n", + " name=\"summary_tool\",\n", + " query_engine=summary_query_engine,\n", + " description=(\n", + " \"Useful if you want to get a summary of MetaGPT\"\n", + " ),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4228ca7c-42a0-494b-987b-5a1c5c584536", + "metadata": { + "height": 98, + "tags": [] + }, + "outputs": [], + "source": [ + "response = llm.predict_and_call(\n", + " [vector_query_tool, summary_tool], \n", + " \"What are the MetaGPT comparisons with ChatDev described on page 8?\", \n", + " verbose=True\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4aa3e8c1-a8c3-4c92-a0e4-5c081f91d966", + "metadata": { + "height": 47, + "tags": [] + }, + "outputs": [], + "source": [ + "for n in response.source_nodes:\n", + " print(n.metadata)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21906d47-7266-4479-bbb4-9f392d5c399b", + "metadata": { + "height": 98, + "tags": [] + }, + "outputs": [], + "source": [ + "response = llm.predict_and_call(\n", + " [vector_query_tool, summary_tool], \n", + " \"What is a summary of the paper?\", \n", + " verbose=True\n", + ")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/building-agentic-rag-with-Llamaindex/l2-tool-calling/helper.py b/building-agentic-rag-with-Llamaindex/l2-tool-calling/helper.py new file mode 100644 index 0000000..adbf81d --- /dev/null +++ b/building-agentic-rag-with-Llamaindex/l2-tool-calling/helper.py @@ -0,0 +1,13 @@ +# Add your utilities or helper functions to this file. + +import os +from dotenv import load_dotenv, find_dotenv + +# these expect to find a .env file at the directory above the lesson. # the format for that file is (without the comment) #API_KEYNAME=AStringThatIsTheLongAPIKeyFromSomeService +def load_env(): + _ = load_dotenv(find_dotenv()) + +def get_openai_api_key(): + load_env() + openai_api_key = os.getenv("OPENAI_API_KEY") + return openai_api_key diff --git a/building-agentic-rag-with-Llamaindex/l2-tool-calling/metagpt.pdf b/building-agentic-rag-with-Llamaindex/l2-tool-calling/metagpt.pdf new file mode 100644 index 0000000..12915b8 Binary files /dev/null and b/building-agentic-rag-with-Llamaindex/l2-tool-calling/metagpt.pdf differ diff --git a/building-agentic-rag-with-Llamaindex/l3-building-agent-reasoning-loop/L3_Building_an_Agent_Reasoning_Loop.ipynb b/building-agentic-rag-with-Llamaindex/l3-building-agent-reasoning-loop/L3_Building_an_Agent_Reasoning_Loop.ipynb new file mode 100644 index 0000000..a8da175 --- /dev/null +++ b/building-agentic-rag-with-Llamaindex/l3-building-agent-reasoning-loop/L3_Building_an_Agent_Reasoning_Loop.ipynb @@ -0,0 +1,355 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "64ec2f0f", + "metadata": {}, + "source": [ + "# Lesson 3: Building an Agent Reasoning Loop" + ] + }, + { + "cell_type": "markdown", + "id": "b0d7f1cf", + "metadata": {}, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b07baa43-7a51-4c39-91cc-aa0d9619b69f", + "metadata": { + "height": 47, + "tags": [] + }, + "outputs": [], + "source": [ + "from helper import get_openai_api_key\n", + "OPENAI_API_KEY = get_openai_api_key()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dcfa86a3-c7f2-41fa-b8b6-5617659ec36a", + "metadata": { + "height": 47, + "tags": [] + }, + "outputs": [], + "source": [ + "import nest_asyncio\n", + "nest_asyncio.apply()" + ] + }, + { + "cell_type": "markdown", + "id": "7d3af4bb", + "metadata": {}, + "source": [ + "## Load the data" + ] + }, + { + "cell_type": "markdown", + "id": "3d8bfb34", + "metadata": { + "tags": [] + }, + "source": [ + "To download this paper, below is the needed code:\n", + "\n", + "#!wget \"https://openreview.net/pdf?id=VtmBAGCN7o\" -O metagpt.pdf\n", + "\n", + "**Note**: The pdf file is included with this lesson. To access it, go to the `File` menu and select`Open...`." + ] + }, + { + "cell_type": "markdown", + "id": "fb741560", + "metadata": {}, + "source": [ + "## Setup the Query Tools" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "77464fb2-5ace-4839-9032-a020df8d4259", + "metadata": { + "height": 64, + "tags": [] + }, + "outputs": [], + "source": [ + "from utils import get_doc_tools\n", + "\n", + "vector_tool, summary_tool = get_doc_tools(\"metagpt.pdf\", \"metagpt\")" + ] + }, + { + "cell_type": "markdown", + "id": "40aae3fc", + "metadata": {}, + "source": [ + "## Setup Function Calling Agent" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ff4f5199-d02c-47b0-a9ab-cf72c8a506a3", + "metadata": { + "height": 64, + "tags": [] + }, + "outputs": [], + "source": [ + "from llama_index.llms.openai import OpenAI\n", + "\n", + "llm = OpenAI(model=\"gpt-3.5-turbo\", temperature=0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9365d78d-8e9f-4f22-8d57-35a4c6aa6baf", + "metadata": { + "height": 166, + "tags": [] + }, + "outputs": [], + "source": [ + "from llama_index.core.agent import FunctionCallingAgentWorker\n", + "from llama_index.core.agent import AgentRunner\n", + "\n", + "agent_worker = FunctionCallingAgentWorker.from_tools(\n", + " [vector_tool, summary_tool], \n", + " llm=llm, \n", + " verbose=True\n", + ")\n", + "agent = AgentRunner(agent_worker)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5a9535d7-0baf-4905-ad16-5fb903d33b85", + "metadata": { + "height": 81, + "tags": [] + }, + "outputs": [], + "source": [ + "response = agent.query(\n", + " \"Tell me about the agent roles in MetaGPT, \"\n", + " \"and then how they communicate with each other.\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bcf74ec4-559f-4284-9ed0-817d26951c54", + "metadata": { + "height": 30, + "tags": [] + }, + "outputs": [], + "source": [ + "print(response.source_nodes[0].get_content(metadata_mode=\"all\"))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b28c184-0b65-4e38-808e-d91a285aaefe", + "metadata": { + "height": 64, + "tags": [] + }, + "outputs": [], + "source": [ + "response = agent.chat(\n", + " \"Tell me about the evaluation datasets used.\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d9586cef-21b5-4732-b95d-619462b4aaf6", + "metadata": { + "height": 30, + "tags": [] + }, + "outputs": [], + "source": [ + "response = agent.chat(\"Tell me the results over one of the above datasets.\")" + ] + }, + { + "cell_type": "markdown", + "id": "1cc4e983", + "metadata": {}, + "source": [ + "## Lower-Level: Debuggability and Control" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "55abad72-b189-471a-accc-1621fd19c804", + "metadata": { + "height": 115, + "tags": [] + }, + "outputs": [], + "source": [ + "agent_worker = FunctionCallingAgentWorker.from_tools(\n", + " [vector_tool, summary_tool], \n", + " llm=llm, \n", + " verbose=True\n", + ")\n", + "agent = AgentRunner(agent_worker)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18e911aa-4640-4f89-99c8-6cdf6aff07c6", + "metadata": { + "height": 81, + "tags": [] + }, + "outputs": [], + "source": [ + "task = agent.create_task(\n", + " \"Tell me about the agent roles in MetaGPT, \"\n", + " \"and then how they communicate with each other.\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5eaf0b88-e03a-4dd9-91f6-f6f0c8758e64", + "metadata": { + "height": 30, + "tags": [] + }, + "outputs": [], + "source": [ + "step_output = agent.run_step(task.task_id)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a8e77fac-8734-4071-a672-b3a9f30e2bf1", + "metadata": { + "height": 64, + "tags": [] + }, + "outputs": [], + "source": [ + "completed_steps = agent.get_completed_steps(task.task_id)\n", + "print(f\"Num completed for task {task.task_id}: {len(completed_steps)}\")\n", + "print(completed_steps[0].output.sources[0].raw_output)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "db8de410-4b82-4daf-93da-28da57cbb0bb", + "metadata": { + "height": 64, + "tags": [] + }, + "outputs": [], + "source": [ + "upcoming_steps = agent.get_upcoming_steps(task.task_id)\n", + "print(f\"Num upcoming steps for task {task.task_id}: {len(upcoming_steps)}\")\n", + "upcoming_steps[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dc352582-2c17-46ef-ba80-0f571e920c3c", + "metadata": { + "height": 64, + "tags": [] + }, + "outputs": [], + "source": [ + "step_output = agent.run_step(\n", + " task.task_id, input=\"What about how agents share information?\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "be80661f-81b1-45fc-b0ba-33a04dae849b", + "metadata": { + "height": 47, + "tags": [] + }, + "outputs": [], + "source": [ + "step_output = agent.run_step(task.task_id)\n", + "print(step_output.is_last)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4496328c-e6d5-4722-a8df-78a73a441b3c", + "metadata": { + "height": 30, + "tags": [] + }, + "outputs": [], + "source": [ + "response = agent.finalize_response(task.task_id)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "601d1bed-78b2-4512-87ac-aec5ce5d8494", + "metadata": { + "height": 30, + "tags": [] + }, + "outputs": [], + "source": [ + "print(str(response))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/building-agentic-rag-with-Llamaindex/l3-building-agent-reasoning-loop/helper.py b/building-agentic-rag-with-Llamaindex/l3-building-agent-reasoning-loop/helper.py new file mode 100644 index 0000000..adbf81d --- /dev/null +++ b/building-agentic-rag-with-Llamaindex/l3-building-agent-reasoning-loop/helper.py @@ -0,0 +1,13 @@ +# Add your utilities or helper functions to this file. + +import os +from dotenv import load_dotenv, find_dotenv + +# these expect to find a .env file at the directory above the lesson. # the format for that file is (without the comment) #API_KEYNAME=AStringThatIsTheLongAPIKeyFromSomeService +def load_env(): + _ = load_dotenv(find_dotenv()) + +def get_openai_api_key(): + load_env() + openai_api_key = os.getenv("OPENAI_API_KEY") + return openai_api_key diff --git a/building-agentic-rag-with-Llamaindex/l3-building-agent-reasoning-loop/utils.py b/building-agentic-rag-with-Llamaindex/l3-building-agent-reasoning-loop/utils.py new file mode 100644 index 0000000..9591560 --- /dev/null +++ b/building-agentic-rag-with-Llamaindex/l3-building-agent-reasoning-loop/utils.py @@ -0,0 +1,73 @@ +# TODO: abstract all of this into a function that takes in a PDF file name + +from llama_index.core import SimpleDirectoryReader, VectorStoreIndex, SummaryIndex +from llama_index.core.node_parser import SentenceSplitter +from llama_index.core.tools import FunctionTool, QueryEngineTool +from llama_index.core.vector_stores import MetadataFilters, FilterCondition +from typing import List, Optional + +def get_doc_tools( + file_path: str, + name: str, +) -> str: + """Get vector query and summary query tools from a document.""" + + # load documents + documents = SimpleDirectoryReader(input_files=[file_path]).load_data() + splitter = SentenceSplitter(chunk_size=1024) + nodes = splitter.get_nodes_from_documents(documents) + vector_index = VectorStoreIndex(nodes) + + def vector_query( + query: str, + page_numbers: Optional[List[str]] = None + ) -> str: + """Use to answer questions over the MetaGPT paper. + + Useful if you have specific questions over the MetaGPT paper. + Always leave page_numbers as None UNLESS there is a specific page you want to search for. + + Args: + query (str): the string query to be embedded. + page_numbers (Optional[List[str]]): Filter by set of pages. Leave as NONE + if we want to perform a vector search + over all pages. Otherwise, filter by the set of specified pages. + + """ + + page_numbers = page_numbers or [] + metadata_dicts = [ + {"key": "page_label", "value": p} for p in page_numbers + ] + + query_engine = vector_index.as_query_engine( + similarity_top_k=2, + filters=MetadataFilters.from_dicts( + metadata_dicts, + condition=FilterCondition.OR + ) + ) + response = query_engine.query(query) + return response + + + vector_query_tool = FunctionTool.from_defaults( + name=f"vector_tool_{name}", + fn=vector_query + ) + + summary_index = SummaryIndex(nodes) + summary_query_engine = summary_index.as_query_engine( + response_mode="tree_summarize", + use_async=True, + ) + summary_tool = QueryEngineTool.from_defaults( + name=f"summary_tool_{name}", + query_engine=summary_query_engine, + description=( + "Use ONLY IF you want to get a holistic summary of MetaGPT. " + "Do NOT use if you have specific questions over MetaGPT." + ), + ) + + return vector_query_tool, summary_tool \ No newline at end of file diff --git a/building-agentic-rag-with-Llamaindex/l4-building-multi-docs-agent/L4_Building_a_Multi-Document_Agent.ipynb b/building-agentic-rag-with-Llamaindex/l4-building-multi-docs-agent/L4_Building_a_Multi-Document_Agent.ipynb new file mode 100644 index 0000000..a6fedd3 --- /dev/null +++ b/building-agentic-rag-with-Llamaindex/l4-building-multi-docs-agent/L4_Building_a_Multi-Document_Agent.ipynb @@ -0,0 +1,451 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0b523e0a", + "metadata": {}, + "source": [ + "# Lesson 4: Building a Multi-Document Agent" + ] + }, + { + "cell_type": "markdown", + "id": "1a323703", + "metadata": {}, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b9625ab2-71b6-4fd0-904e-42df80d3215f", + "metadata": { + "height": 47, + "tags": [] + }, + "outputs": [], + "source": [ + "from helper import get_openai_api_key\n", + "OPENAI_API_KEY = get_openai_api_key()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3221a474-5817-4db2-af46-e029042a75a5", + "metadata": { + "height": 47, + "tags": [] + }, + "outputs": [], + "source": [ + "import nest_asyncio\n", + "nest_asyncio.apply()" + ] + }, + { + "cell_type": "markdown", + "id": "20adaa26", + "metadata": {}, + "source": [ + "## 1. Setup an agent over 3 papers" + ] + }, + { + "cell_type": "markdown", + "id": "48b71ff6", + "metadata": {}, + "source": [ + "**Note**: The pdf files are included with this lesson. To access these papers, go to the `File` menu and select`Open...`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ed10a24b-d65c-4b98-a93a-94ccdb8900d0", + "metadata": { + "height": 200, + "tags": [] + }, + "outputs": [], + "source": [ + "urls = [\n", + " \"https://openreview.net/pdf?id=VtmBAGCN7o\",\n", + " \"https://openreview.net/pdf?id=6PmJoRfdaK\",\n", + " \"https://openreview.net/pdf?id=hSyW5go0v8\",\n", + "]\n", + "\n", + "papers = [\n", + " \"metagpt.pdf\",\n", + " \"longlora.pdf\",\n", + " \"selfrag.pdf\",\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0d8f3185-3221-4b00-bd38-41d36e4a3307", + "metadata": { + "height": 149, + "tags": [] + }, + "outputs": [], + "source": [ + "from utils import get_doc_tools\n", + "from pathlib import Path\n", + "\n", + "paper_to_tools_dict = {}\n", + "for paper in papers:\n", + " print(f\"Getting tools for paper: {paper}\")\n", + " vector_tool, summary_tool = get_doc_tools(paper, Path(paper).stem)\n", + " paper_to_tools_dict[paper] = [vector_tool, summary_tool]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0e541bdd-14e1-41b6-81b5-b1bfda078d07", + "metadata": { + "height": 30, + "tags": [] + }, + "outputs": [], + "source": [ + "initial_tools = [t for paper in papers for t in paper_to_tools_dict[paper]]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bff58c52", + "metadata": { + "height": 64 + }, + "outputs": [], + "source": [ + "from llama_index.llms.openai import OpenAI\n", + "\n", + "llm = OpenAI(model=\"gpt-3.5-turbo\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2f2c6a9f", + "metadata": { + "height": 30 + }, + "outputs": [], + "source": [ + "len(initial_tools)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a124a438-5609-402e-8642-69d1088cb9ad", + "metadata": { + "height": 166, + "tags": [] + }, + "outputs": [], + "source": [ + "from llama_index.core.agent import FunctionCallingAgentWorker\n", + "from llama_index.core.agent import AgentRunner\n", + "\n", + "agent_worker = FunctionCallingAgentWorker.from_tools(\n", + " initial_tools, \n", + " llm=llm, \n", + " verbose=True\n", + ")\n", + "agent = AgentRunner(agent_worker)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17409d4c-05a9-4bf4-b74f-75135fa3cb6b", + "metadata": { + "height": 81, + "tags": [] + }, + "outputs": [], + "source": [ + "response = agent.query(\n", + " \"Tell me about the evaluation dataset used in LongLoRA, \"\n", + " \"and then tell me about the evaluation results\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ace340b1-761f-4058-be41-68cf131541e4", + "metadata": { + "height": 47, + "tags": [] + }, + "outputs": [], + "source": [ + "response = agent.query(\"Give me a summary of both Self-RAG and LongLoRA\")\n", + "print(str(response))" + ] + }, + { + "cell_type": "markdown", + "id": "7eede70c", + "metadata": {}, + "source": [ + "## 2. Setup an agent over 11 papers" + ] + }, + { + "cell_type": "markdown", + "id": "18771e69", + "metadata": {}, + "source": [ + "### Download 11 ICLR papers" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "60d01d2c-547f-4054-b0fe-ed9b1a9cc3b5", + "metadata": { + "height": 472, + "tags": [] + }, + "outputs": [], + "source": [ + "urls = [\n", + " \"https://openreview.net/pdf?id=VtmBAGCN7o\",\n", + " \"https://openreview.net/pdf?id=6PmJoRfdaK\",\n", + " \"https://openreview.net/pdf?id=LzPWWPAdY4\",\n", + " \"https://openreview.net/pdf?id=VTF8yNQM66\",\n", + " \"https://openreview.net/pdf?id=hSyW5go0v8\",\n", + " \"https://openreview.net/pdf?id=9WD9KwssyT\",\n", + " \"https://openreview.net/pdf?id=yV6fD7LYkF\",\n", + " \"https://openreview.net/pdf?id=hnrB5YHoYu\",\n", + " \"https://openreview.net/pdf?id=WbWtOYIzIK\",\n", + " \"https://openreview.net/pdf?id=c5pwL0Soay\",\n", + " \"https://openreview.net/pdf?id=TpD2aG1h0D\"\n", + "]\n", + "\n", + "papers = [\n", + " \"metagpt.pdf\",\n", + " \"longlora.pdf\",\n", + " \"loftq.pdf\",\n", + " \"swebench.pdf\",\n", + " \"selfrag.pdf\",\n", + " \"zipformer.pdf\",\n", + " \"values.pdf\",\n", + " \"finetune_fair_diffusion.pdf\",\n", + " \"knowledge_card.pdf\",\n", + " \"metra.pdf\",\n", + " \"vr_mcl.pdf\"\n", + "]" + ] + }, + { + "cell_type": "markdown", + "id": "b77426cb", + "metadata": { + "tags": [] + }, + "source": [ + "To download these papers, below is the needed code:\n", + "\n", + "\n", + " #for url, paper in zip(urls, papers):\n", + " #!wget \"{url}\" -O \"{paper}\"\n", + " \n", + " \n", + "**Note**: The pdf files are included with this lesson. To access these papers, go to the `File` menu and select`Open...`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ea5ee34d-02ac-4537-ae20-7ef6c5767172", + "metadata": { + "height": 149, + "tags": [] + }, + "outputs": [], + "source": [ + "from utils import get_doc_tools\n", + "from pathlib import Path\n", + "\n", + "paper_to_tools_dict = {}\n", + "for paper in papers:\n", + " print(f\"Getting tools for paper: {paper}\")\n", + " vector_tool, summary_tool = get_doc_tools(paper, Path(paper).stem)\n", + " paper_to_tools_dict[paper] = [vector_tool, summary_tool]" + ] + }, + { + "cell_type": "markdown", + "id": "4e35d52c", + "metadata": {}, + "source": [ + "### Extend the Agent with Tool Retrieval" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20154923-873e-4941-9a3a-4926ab5f9b8c", + "metadata": { + "height": 30, + "tags": [] + }, + "outputs": [], + "source": [ + "all_tools = [t for paper in papers for t in paper_to_tools_dict[paper]]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "671582f9-70d7-4a8f-b813-58b2a068ca72", + "metadata": { + "height": 149, + "tags": [] + }, + "outputs": [], + "source": [ + "# define an \"object\" index and retriever over these tools\n", + "from llama_index.core import VectorStoreIndex\n", + "from llama_index.core.objects import ObjectIndex\n", + "\n", + "obj_index = ObjectIndex.from_objects(\n", + " all_tools,\n", + " index_cls=VectorStoreIndex,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3929882-e9dc-46ca-b495-53e3ed60340e", + "metadata": { + "height": 30, + "tags": [] + }, + "outputs": [], + "source": [ + "obj_retriever = obj_index.as_retriever(similarity_top_k=3)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ba9cfecd-fe14-4da8-b9ba-b3d485d98a03", + "metadata": { + "height": 64, + "tags": [] + }, + "outputs": [], + "source": [ + "tools = obj_retriever.retrieve(\n", + " \"Tell me about the eval dataset used in MetaGPT and SWE-Bench\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c270ffbf-69c7-48ea-a028-9ba25221cde5", + "metadata": { + "height": 30, + "tags": [] + }, + "outputs": [], + "source": [ + "tools[2].metadata" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9cc0a0b6-9858-4348-9ae0-1cd4160f3fb7", + "metadata": { + "height": 251, + "tags": [] + }, + "outputs": [], + "source": [ + "from llama_index.core.agent import FunctionCallingAgentWorker\n", + "from llama_index.core.agent import AgentRunner\n", + "\n", + "agent_worker = FunctionCallingAgentWorker.from_tools(\n", + " tool_retriever=obj_retriever,\n", + " llm=llm, \n", + " system_prompt=\"\"\" \\\n", + "You are an agent designed to answer queries over a set of given papers.\n", + "Please always use the tools provided to answer a question. Do not rely on prior knowledge.\\\n", + "\n", + "\"\"\",\n", + " verbose=True\n", + ")\n", + "agent = AgentRunner(agent_worker)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a250cf1a-e011-4994-bcca-4e0294f20864", + "metadata": { + "height": 98, + "tags": [] + }, + "outputs": [], + "source": [ + "response = agent.query(\n", + " \"Tell me about the evaluation dataset used \"\n", + " \"in MetaGPT and compare it against SWE-Bench\"\n", + ")\n", + "print(str(response))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8084c8cb-98ed-4835-aaa4-5b0c7254be6d", + "metadata": { + "height": 81, + "tags": [] + }, + "outputs": [], + "source": [ + "response = agent.query(\n", + " \"Compare and contrast the LoRA papers (LongLoRA, LoftQ). \"\n", + " \"Analyze the approach in each paper first. \"\n", + ")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/building-agentic-rag-with-Llamaindex/l4-building-multi-docs-agent/utils.py b/building-agentic-rag-with-Llamaindex/l4-building-multi-docs-agent/utils.py new file mode 100644 index 0000000..00866e7 --- /dev/null +++ b/building-agentic-rag-with-Llamaindex/l4-building-multi-docs-agent/utils.py @@ -0,0 +1,132 @@ +# TODO: abstract all of this into a function that takes in a PDF file name + +from llama_index.core import SimpleDirectoryReader, VectorStoreIndex, SummaryIndex +from llama_index.core.node_parser import SentenceSplitter +from llama_index.core.tools import FunctionTool, QueryEngineTool +from llama_index.core.vector_stores import MetadataFilters, FilterCondition +from typing import List, Optional + + +# def get_doc_tools( +# file_path: str, +# name: str, +# ) -> str: +# """Get vector query and summary query tools from a document.""" + +# # load documents +# documents = SimpleDirectoryReader(input_files=[file_path]).load_data() +# splitter = SentenceSplitter(chunk_size=1024) +# nodes = splitter.get_nodes_from_documents(documents) +# vector_index = VectorStoreIndex(nodes) + +# def vector_query( +# query: str, +# filter_key_list: List[str], +# filter_value_list: List[str] +# ) -> str: +# """Perform a vector search over an index. + +# query (str): the string query to be embedded. +# filter_key_list (List[str]): A list of metadata filter field names +# Must specify ['page_label'] or empty list. Please leave empty +# if there are no explicit filters to specify. +# filter_value_list (List[str]): List of metadata filter field values +# (corresponding to names specified in filter_key_list) + +# """ +# metadata_dicts = [ +# {"key": k, "value": v} for k, v in zip(filter_key_list, filter_value_list) +# ] + +# query_engine = vector_index.as_query_engine( +# similarity_top_k=2, +# filters=MetadataFilters.from_dicts(metadata_dicts) +# ) +# response = query_engine.query(query) +# return response + +# vector_query_tool = FunctionTool.from_defaults( +# fn=vector_query, +# name=f"vector_query_{name}" +# ) + +# summary_index = SummaryIndex(nodes) +# summary_query_engine = summary_index.as_query_engine( +# response_mode="tree_summarize", +# use_async=True, +# ) +# summary_tool = QueryEngineTool.from_defaults( +# query_engine=summary_query_engine, +# name=f"summary_query_{name}", +# description=( +# f"Useful for summarization questions related to {name}" +# ), +# ) +# return vector_query_tool, summary_tool + + + +def get_doc_tools( + file_path: str, + name: str, +) -> str: + """Get vector query and summary query tools from a document.""" + + # load documents + documents = SimpleDirectoryReader(input_files=[file_path]).load_data() + splitter = SentenceSplitter(chunk_size=1024) + nodes = splitter.get_nodes_from_documents(documents) + vector_index = VectorStoreIndex(nodes) + + def vector_query( + query: str, + page_numbers: Optional[List[str]] = None + ) -> str: + """Use to answer questions over a given paper. + + Useful if you have specific questions over the paper. + Always leave page_numbers as None UNLESS there is a specific page you want to search for. + + Args: + query (str): the string query to be embedded. + page_numbers (Optional[List[str]]): Filter by set of pages. Leave as NONE + if we want to perform a vector search + over all pages. Otherwise, filter by the set of specified pages. + + """ + + page_numbers = page_numbers or [] + metadata_dicts = [ + {"key": "page_label", "value": p} for p in page_numbers + ] + + query_engine = vector_index.as_query_engine( + similarity_top_k=2, + filters=MetadataFilters.from_dicts( + metadata_dicts, + condition=FilterCondition.OR + ) + ) + response = query_engine.query(query) + return response + + + vector_query_tool = FunctionTool.from_defaults( + name=f"vector_tool_{name}", + fn=vector_query + ) + + summary_index = SummaryIndex(nodes) + summary_query_engine = summary_index.as_query_engine( + response_mode="tree_summarize", + use_async=True, + ) + summary_tool = QueryEngineTool.from_defaults( + name=f"summary_tool_{name}", + query_engine=summary_query_engine, + description=( + f"Useful for summarization questions related to {name}" + ), + ) + + return vector_query_tool, summary_tool \ No newline at end of file diff --git a/building-agentic-rag-with-Llamaindex/summary-index.png b/building-agentic-rag-with-Llamaindex/summary-index.png new file mode 100644 index 0000000..ec66353 Binary files /dev/null and b/building-agentic-rag-with-Llamaindex/summary-index.png differ diff --git a/building-agentic-rag-with-Llamaindex/three-documents-agent.png b/building-agentic-rag-with-Llamaindex/three-documents-agent.png new file mode 100644 index 0000000..add2529 Binary files /dev/null and b/building-agentic-rag-with-Llamaindex/three-documents-agent.png differ