Merge branch 'main' into dqbd/generator-traceable

.github/actions/js-integration-tests/action.yml

@@ -7,6 +7,9 @@ inputs:
   langchain-api-key:
     description: "Langchain"
     required: true
+  langchain-endpoint:
+    description: "LangSmith Endpoint"
+    required: true
   openai-api-key:
     description: "OpenAI API key"
     required: false
@@ -34,5 +37,6 @@ runs:
       working-directory: js
       env:
         LANGCHAIN_TRACING_V2: "true"
+        LANGCHAIN_ENDPOINT: ${{ inputs.langchain-endpoint }}
         LANGCHAIN_API_KEY: ${{ inputs.langchain-api-key }}
         OPENAI_API_KEY: ${{ inputs.openai-api-key }}

.github/actions/python-integration-tests/action.yml

@@ -7,6 +7,9 @@ inputs:
   langchain-api-key:
     description: "Langchain"
     required: true
+  langchain-endpoint:
+    description: "LangSmith Endpoint"
+    required: true
   openai-api-key:
     description: "OpenAI API key"
     required: false
@@ -40,6 +43,7 @@ runs:
     - name: Run integration tests
       env:
         LANGCHAIN_TRACING_V2: "true"
+        LANGCHAIN_ENDPOINT: ${{ inputs.langchain-endpoint }}
         LANGCHAIN_API_KEY: ${{ inputs.langchain-api-key }}
         OPENAI_API_KEY: ${{ inputs.openai-api-key }}
       run: make integration_tests_fast
@@ -49,6 +53,7 @@ runs:
     - name: Run doctest
       env:
         LANGCHAIN_TRACING_V2: "true"
+        LANGCHAIN_ENDPOINT: ${{ inputs.langchain-endpoint }}
         LANGCHAIN_API_KEY: ${{ inputs.langchain-api-key }}
         OPENAI_API_KEY: ${{ inputs.openai-api-key }}
         ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }}
@@ -60,6 +65,7 @@ runs:
     - name: Run Evaluation
       env:
           LANGCHAIN_TRACING_V2: "true"
+          LANGCHAIN_ENDPOINT: ${{ inputs.langchain-endpoint }}
           LANGCHAIN_API_KEY: ${{ inputs.langchain-api-key }}
           OPENAI_API_KEY: ${{ inputs.openai-api-key }}
           ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }}

.github/workflows/integration_tests.yml

@@ -47,7 +47,8 @@ jobs:
         uses: ./.github/actions/python-integration-tests
         with:
           python-version: 3.11
-          langchain-api-key: ${{ secrets.LANGCHAIN_API_KEY }}
+          langchain-endpoint: https://api.smith.langchain.com
+          langchain-api-key: ${{ secrets.LANGSMITH_API_KEY }}
           openai-api-key: ${{ secrets.OPENAI_API_KEY }}
           anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}
 
@@ -75,5 +76,6 @@ jobs:
         uses: ./.github/actions/js-integration-tests
         with:
           node-version: 20.x
-          langchain-api-key: ${{ secrets.LANGCHAIN_API_KEY }}
+          langchain-endpoint: https://api.smith.langchain.com
+          langchain-api-key: ${{ secrets.LANGSMITH_API_KEY }}
           openai-api-key: ${{ secrets.OPENAI_API_KEY }}

js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.1.23-rc.1",
+  "version": "0.1.23",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "[email protected]",
   "files": [

js/src/index.ts

@@ -11,4 +11,4 @@ export type {
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
 
 // Update using yarn bump-version
-export const __version__ = "0.1.23-rc.1";
+export const __version__ = "0.1.23";

js/src/tests/traceable.test.ts

@@ -1,4 +1,4 @@
-import type { RunTree } from "../run_trees.js";
+import type { RunTree, RunTreeConfig } from "../run_trees.js";
 import { ROOT, traceable } from "../traceable.js";
 import { getAssumedTreeFromCalls } from "./utils/tree.js";
 import { mockClient } from "./utils/mock_client.js";
@@ -763,3 +763,69 @@ describe("generator", () => {
     });
   });
 });
+
+test("metadata", async () => {
+  const { client, callSpy } = mockClient();
+  const main = traceable(async (): Promise<number> => 42, {
+    client,
+    name: "main",
+    metadata: { customValue: "hello" },
+    tracingEnabled: true,
+  });
+
+  await main();
+
+  expect(getAssumedTreeFromCalls(callSpy.mock.calls)).toMatchObject({
+    nodes: ["main:0"],
+    edges: [],
+    data: {
+      "main:0": {
+        extra: { metadata: { customValue: "hello" } },
+        outputs: { outputs: 42 },
+      },
+    },
+  });
+});
+
+test("argsConfigPath", async () => {
+  const { client, callSpy } = mockClient();
+  const main = traceable(
+    async (
+      value: number,
+      options: {
+        suffix: string;
+        langsmithExtra?: Partial<RunTreeConfig>;
+      }
+    ): Promise<string> => `${value}${options.suffix}`,
+    {
+      client,
+      name: "main",
+      argsConfigPath: [1, "langsmithExtra"],
+      tracingEnabled: true,
+    }
+  );
+
+  await main(1, {
+    suffix: "hello",
+    langsmithExtra: {
+      name: "renamed",
+      tags: ["tag1", "tag2"],
+      metadata: { customValue: "hello" },
+    },
+  });
+
+  expect(getAssumedTreeFromCalls(callSpy.mock.calls)).toMatchObject({
+    nodes: ["renamed:0"],
+    edges: [],
+    data: {
+      "renamed:0": {
+        extra: { metadata: { customValue: "hello" } },
+        tags: ["tag1", "tag2"],
+        inputs: {
+          args: [1, { suffix: "hello" }],
+        },
+        outputs: { outputs: "1hello" },
+      },
+    },
+  });
+});

js/src/tests/wrapped_openai.int.test.ts

@@ -4,6 +4,8 @@ import { jest } from "@jest/globals";
 import { OpenAI } from "openai";
 import { wrapOpenAI } from "../wrappers/index.js";
 import { Client } from "../client.js";
+import { mockClient } from "./utils/mock_client.js";
+import { getAssumedTreeFromCalls } from "./utils/tree.js";
 
 test("wrapOpenAI should return type compatible with OpenAI", async () => {
   let originalClient = new OpenAI();
@@ -462,3 +464,69 @@ test.skip("no tracing with env var unset", async () => {
   expect(patched).toBeDefined();
   console.log(patched);
 });
+
+test("wrapping same instance", async () => {
+  const wrapped = wrapOpenAI(new OpenAI());
+  expect(() => wrapOpenAI(wrapped)).toThrowError(
+    "This instance of OpenAI client has been already wrapped once."
+  );
+});
+
+test("chat.concurrent extra name", async () => {
+  const { client, callSpy } = mockClient();
+
+  const openai = wrapOpenAI(new OpenAI(), {
+    client,
+  });
+
+  await openai.chat.completions.create(
+    {
+      messages: [{ role: "user", content: `Say 'red'` }],
+      temperature: 0,
+      seed: 42,
+      model: "gpt-3.5-turbo",
+    },
+    { langsmithExtra: { name: "red", metadata: { customKey: "red" } } }
+  );
+
+  const stream = await openai.chat.completions.create(
+    {
+      messages: [{ role: "user", content: `Say 'green'` }],
+      temperature: 0,
+      seed: 42,
+      model: "gpt-3.5-turbo",
+      stream: true,
+    },
+    { langsmithExtra: { name: "green", metadata: { customKey: "green" } } }
+  );
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  for await (const _ of stream) {
+    // pass
+  }
+
+  expect(getAssumedTreeFromCalls(callSpy.mock.calls)).toMatchObject({
+    nodes: ["red:0", "green:1"],
+    edges: [],
+    data: {
+      "red:0": {
+        name: "red",
+        extra: { metadata: { customKey: "red" } },
+        outputs: {
+          choices: [
+            { index: 0, message: { role: "assistant", content: "Red" } },
+          ],
+        },
+      },
+      "green:1": {
+        name: "green",
+        extra: { metadata: { customKey: "green" } },
+        outputs: {
+          choices: [
+            { index: 0, message: { role: "assistant", content: "Green" } },
+          ],
+        },
+      },
+    },
+  });
+});

js/src/wrappers/openai.ts

@@ -1,8 +1,7 @@
 import { OpenAI } from "openai";
 import type { APIPromise } from "openai/core";
 import type { Client, RunTreeConfig } from "../index.js";
-import { type RunnableConfigLike } from "../run_trees.js";
-import { traceable, type RunTreeLike } from "../traceable.js";
+import { isTraceableFunction, traceable } from "../traceable.js";
 
 // Extra leniency around types in case multiple OpenAI SDK versions get installed
 type OpenAIType = {
@@ -18,22 +17,23 @@ type OpenAIType = {
   };
 };
 
+type ExtraRunTreeConfig = Pick<
+  Partial<RunTreeConfig>,
+  "name" | "metadata" | "tags"
+>;
+
 type PatchedOpenAIClient<T extends OpenAIType> = T & {
   chat: T["chat"] & {
     completions: T["chat"]["completions"] & {
       create: {
         (
           arg: OpenAI.ChatCompletionCreateParamsStreaming,
-          arg2?: OpenAI.RequestOptions & {
-            langsmithExtra?: RunnableConfigLike | RunTreeLike;
-          }
+          arg2?: OpenAI.RequestOptions & { langsmithExtra?: ExtraRunTreeConfig }
         ): APIPromise<AsyncGenerator<OpenAI.ChatCompletionChunk>>;
       } & {
         (
           arg: OpenAI.ChatCompletionCreateParamsNonStreaming,
-          arg2?: OpenAI.RequestOptions & {
-            langsmithExtra?: RunnableConfigLike | RunTreeLike;
-          }
+          arg2?: OpenAI.RequestOptions & { langsmithExtra?: ExtraRunTreeConfig }
         ): APIPromise<OpenAI.ChatCompletionChunk>;
       };
     };
@@ -42,16 +42,12 @@ type PatchedOpenAIClient<T extends OpenAIType> = T & {
     create: {
       (
         arg: OpenAI.CompletionCreateParamsStreaming,
-        arg2?: OpenAI.RequestOptions & {
-          langsmithExtra?: RunnableConfigLike | RunTreeLike;
-        }
+        arg2?: OpenAI.RequestOptions & { langsmithExtra?: ExtraRunTreeConfig }
       ): APIPromise<AsyncGenerator<OpenAI.Completion>>;
     } & {
       (
         arg: OpenAI.CompletionCreateParamsNonStreaming,
-        arg2?: OpenAI.RequestOptions & {
-          langsmithExtra?: RunnableConfigLike | RunTreeLike;
-        }
+        arg2?: OpenAI.RequestOptions & { langsmithExtra?: ExtraRunTreeConfig }
       ): APIPromise<OpenAI.Completion>;
     };
   };
@@ -211,6 +207,15 @@ export const wrapOpenAI = <T extends OpenAIType>(
   openai: T,
   options?: Partial<RunTreeConfig>
 ): PatchedOpenAIClient<T> => {
+  if (
+    isTraceableFunction(openai.chat.completions.create) ||
+    isTraceableFunction(openai.completions.create)
+  ) {
+    throw new Error(
+      "This instance of OpenAI client has been already wrapped once."
+    );
+  }
+
   openai.chat.completions.create = traceable(
     openai.chat.completions.create.bind(openai.chat.completions),
     {