From 8690bc304ae9e4e36d2865966a0f879c407975ee Mon Sep 17 00:00:00 2001 From: Boris Feld Date: Tue, 23 Jan 2024 16:44:03 +0100 Subject: [PATCH 1/4] Update minimal Comet version for FastAI --- .../model-training/fastai/fastai-hello-world/requirements.txt | 2 +- .../model-training/fastai/notebooks/fastai_hello_world.ipynb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/integrations/model-training/fastai/fastai-hello-world/requirements.txt b/integrations/model-training/fastai/fastai-hello-world/requirements.txt index 10f5d232..944d34f1 100644 --- a/integrations/model-training/fastai/fastai-hello-world/requirements.txt +++ b/integrations/model-training/fastai/fastai-hello-world/requirements.txt @@ -1,2 +1,2 @@ -comet_ml +comet_ml>=3.37.0 fastai diff --git a/integrations/model-training/fastai/notebooks/fastai_hello_world.ipynb b/integrations/model-training/fastai/notebooks/fastai_hello_world.ipynb index 47619542..ad6194af 100644 --- a/integrations/model-training/fastai/notebooks/fastai_hello_world.ipynb +++ b/integrations/model-training/fastai/notebooks/fastai_hello_world.ipynb @@ -43,7 +43,7 @@ }, "outputs": [], "source": [ - "%pip install -U fastai comet_ml" + "%pip install -U fastai comet_ml>=3.37.0" ] }, { From 8944b52b67c43feab5dcbf4c580437345f4be3d0 Mon Sep 17 00:00:00 2001 From: Boris Feld Date: Tue, 23 Jan 2024 17:07:04 +0100 Subject: [PATCH 2/4] Remove old content in the fastai notebook --- .../model-training/fastai/notebooks/fastai_hello_world.ipynb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/integrations/model-training/fastai/notebooks/fastai_hello_world.ipynb b/integrations/model-training/fastai/notebooks/fastai_hello_world.ipynb index ad6194af..9f26de8f 100644 --- a/integrations/model-training/fastai/notebooks/fastai_hello_world.ipynb +++ b/integrations/model-training/fastai/notebooks/fastai_hello_world.ipynb @@ -204,9 +204,7 @@ "id": "q-LxaKnwmKd2" }, "source": [ - "## 5. Training\n", - "\n", - "In fastai, we can train differently depending on if we are running CPU or a GPU. To test, we can use the `data.device.type` property. This will create a fastai `Learner`:" + "## 5. Training" ] }, { From 3c355e870f4dd0b6e5a54fb15021558e3c3a3f26 Mon Sep 17 00:00:00 2001 From: Boris Feld Date: Fri, 26 Jan 2024 17:06:51 +0100 Subject: [PATCH 3/4] Update Langchain example with new tracing mechanism (#155) * Update Langchain example with the first beta version of the new Langchain integration * Use comet_llm.init * Update langchain example for new tracer mechanism --- .../notebooks/Comet_with_Langchain.ipynb | 88 +++++++++---------- 1 file changed, 42 insertions(+), 46 deletions(-) diff --git a/integrations/llm/langchain/notebooks/Comet_with_Langchain.ipynb b/integrations/llm/langchain/notebooks/Comet_with_Langchain.ipynb index 783a2607..55afa5e0 100644 --- a/integrations/llm/langchain/notebooks/Comet_with_Langchain.ipynb +++ b/integrations/llm/langchain/notebooks/Comet_with_Langchain.ipynb @@ -18,13 +18,10 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "In this guide we will demonstrate how to track your Langchain Experiments, Evaluation Metrics, and LLM Sessions with [Comet](https://www.comet.com/site/?utm_source=langchain&utm_medium=referral&utm_campaign=comet_notebook). \n", + "In this guide we will demonstrate how to track your Langchain prompts, Chains, and Agents with [Comet](https://www.comet.com/site/?utm_source=langchain&utm_medium=referral&utm_campaign=comet_notebook).\n", "\n", - "\n", - " \"Open\n", - "\n", "\n", - "**Example Project:** [Comet with LangChain](https://www.comet.com/examples/comet-example-langchain-notebook/prompts?utm_source=langchain&utm_medium=referral&utm_campaign=comet_notebook)" + "**Example Project:** [Comet with LangChain](https://www.comet.com/examples/comet-example-langchain-llm-notebook/prompts?utm_source=langchain&utm_medium=referral&utm_campaign=comet_notebook)" ] }, { @@ -47,7 +44,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U comet_llm \"langchain>=0.0.346\" openai numexpr" + "%pip install -U comet_llm \"langchain>=0.1.3\" \"langchain-openai\" openai numexpr" ] }, { @@ -57,13 +54,6 @@ "### Initialize Comet and Set your Credentials" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can grab your [Comet API Key here](https://www.comet.com/signup?utm_source=langchain&utm_medium=referral&utm_campaign=comet_notebook) or click the link after intializing Comet" - ] - }, { "cell_type": "code", "execution_count": null, @@ -72,7 +62,7 @@ "source": [ "import comet_llm\n", "\n", - "comet_llm.init(project=\"comet-example-langchain-notebook\")" + "comet_llm.init(project=\"comet-example-langchain-llm-notebook\")\n", ] }, { @@ -97,10 +87,32 @@ "source": [ "import os\n", "\n", - "os.environ[\"OPENAI_API_KEY\"] = \"...\"\n", + "# os.environ[\"OPENAI_API_KEY\"] = \"...\"\n", "# os.environ[\"OPENAI_ORGANIZATION\"] = \"...\"" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Tracing with Comet\n", + "\n", + "There are two ways to trace your LangChains executions with Comet:\n", + "\n", + "1. Setting the `LANGCHAIN_COMET_TRACING` environment variable to \"true\". This is the recommended way.\n", + "2. Import the `CometTracer` manually and pass it explicitely." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "os.environ[\"LANGCHAIN_COMET_TRACING\"] = \"true\"\n", + "from langchain.callbacks.tracers.comet import CometTracer" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -116,17 +128,13 @@ "source": [ "from datetime import datetime\n", "\n", - "from langchain.callbacks.tracers.comet import CometTracer\n", - "from langchain.llms import OpenAI\n", + "from langchain_openai import OpenAI\n", "\n", - "comet_callback = CometTracer()\n", "llm = OpenAI(temperature=0.9, verbose=True)\n", - "llm_result = llm.generate(\n", - " [\"Tell me a joke\", \"Tell me a poem\", \"Tell me a fact\"] * 3,\n", - " callbacks=[comet_callback],\n", - ")\n", - "print(\"LLM result\", llm_result)\n", - "comet_callback.flush()" + "\n", + "for prompt in [\"Tell me a joke\", \"Tell me a poem\", \"Tell me a fact\"]:\n", + " llm_result = llm.generate([prompt])\n", + " print(\"LLM result\", llm_result)" ] }, { @@ -142,25 +150,20 @@ "metadata": {}, "outputs": [], "source": [ - "from langchain.callbacks.tracers.comet import CometTracer\n", "from langchain.chains import LLMChain\n", - "from langchain.llms import OpenAI\n", + "from langchain_openai import OpenAI\n", "from langchain.prompts import PromptTemplate\n", "\n", - "comet_callback = CometTracer()\n", - "callbacks = [comet_callback]\n", - "\n", "llm = OpenAI(temperature=0.9, verbose=True)\n", "\n", "template = \"\"\"You are a playwright. Given the title of play, it is your job to write a synopsis for that title.\n", "Title: {title}\n", "Playwright: This is a synopsis for the above play:\"\"\"\n", "prompt_template = PromptTemplate(input_variables=[\"title\"], template=template)\n", - "synopsis_chain = LLMChain(llm=llm, prompt=prompt_template, verbose=True)\n", + "synopsis_chain = LLMChain(llm=llm, prompt=prompt_template)\n", "\n", "test_prompts = [{\"title\": \"Documentary about Bigfoot in Paris\"}]\n", - "print(synopsis_chain.apply(test_prompts, callbacks=callbacks))\n", - "comet_callback.flush()" + "print(synopsis_chain.apply(test_prompts))" ] }, { @@ -176,27 +179,20 @@ "metadata": {}, "outputs": [], "source": [ - "from langchain.agents import AgentType, initialize_agent, load_tools\n", - "from langchain.callbacks.tracers.comet import CometTracer\n", - "from langchain.llms import OpenAI\n", + "from langchain.agents import initialize_agent, load_tools\n", + "from langchain_openai import OpenAI\n", "\n", - "comet_callback = CometTracer()\n", - "callbacks = [comet_callback]\n", + "llm = OpenAI(temperature=0.9)\n", "\n", - "llm = OpenAI(temperature=0.9, verbose=True)\n", - "\n", - "tools = load_tools([\"llm-math\"], llm=llm, verbose=True)\n", + "tools = load_tools([\"llm-math\"], llm=llm)\n", "agent = initialize_agent(\n", " tools,\n", " llm,\n", - " AgentType.ZERO_SHOT_REACT_DESCRIPTION,\n", - " verbose=True,\n", + " agent=\"zero-shot-react-description\",\n", ")\n", "agent.run(\n", " \"What is 2 raised to .123243 power?\",\n", - " callbacks=callbacks,\n", - ")\n", - "comet_callback.flush()" + ")" ] } ], @@ -216,7 +212,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.3" + "version": "3.10.12" } }, "nbformat": 4, From 21801076f5ae8527689b91aa601f09962e4aff8f Mon Sep 17 00:00:00 2001 From: Boris Feld Date: Fri, 26 Jan 2024 17:37:04 +0100 Subject: [PATCH 4/4] Fix invalid JSON after merge --- integrations/llm/langchain/notebooks/Comet_with_Langchain.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/llm/langchain/notebooks/Comet_with_Langchain.ipynb b/integrations/llm/langchain/notebooks/Comet_with_Langchain.ipynb index 55afa5e0..3cd8177e 100644 --- a/integrations/llm/langchain/notebooks/Comet_with_Langchain.ipynb +++ b/integrations/llm/langchain/notebooks/Comet_with_Langchain.ipynb @@ -62,7 +62,7 @@ "source": [ "import comet_llm\n", "\n", - "comet_llm.init(project=\"comet-example-langchain-llm-notebook\")\n", + "comet_llm.init(project=\"comet-example-langchain-llm-notebook\")" ] }, {