From 82cd2bb8dacda186302696b6bc64491887dbfa84 Mon Sep 17 00:00:00 2001 From: bracesproul Date: Wed, 12 Jun 2024 18:19:07 -0700 Subject: [PATCH 1/5] core[minor]: Add base implementation of withStructuredOutput --- .github/workflows/standard-tests.yml | 4 +- langchain-core/.gitignore | 4 + langchain-core/langchain.config.js | 1 + langchain-core/package.json | 13 ++ .../src/language_models/chat_models.ts | 157 +++++++++++++++++- langchain-core/src/utils/is_openai_tool.ts | 17 ++ 6 files changed, 189 insertions(+), 7 deletions(-) create mode 100644 langchain-core/src/utils/is_openai_tool.ts diff --git a/.github/workflows/standard-tests.yml b/.github/workflows/standard-tests.yml index 24bd8fb9b557..221837ef7448 100644 --- a/.github/workflows/standard-tests.yml +++ b/.github/workflows/standard-tests.yml @@ -28,9 +28,9 @@ jobs: strategy: matrix: package: [anthropic, cohere, google-genai, groq, mistralai] - if: contains(needs.get-changed-files.outputs.changed_files, 'langchain-core/') || contains(needs.get-changed-files.outputs.changed_files, 'libs/langchain-${{ matrix.package }}/') steps: - uses: actions/checkout@v4 + if: contains(needs.get-changed-files.outputs.changed_files, 'langchain-core/') || contains(needs.get-changed-files.outputs.changed_files, 'libs/langchain-${{ matrix.package }}/') - name: Use Node.js 18.x uses: actions/setup-node@v3 with: @@ -113,4 +113,4 @@ jobs: env: BEDROCK_AWS_REGION: "us-east-1" BEDROCK_AWS_SECRET_ACCESS_KEY: ${{ secrets.BEDROCK_AWS_SECRET_ACCESS_KEY }} - BEDROCK_AWS_ACCESS_KEY_ID: ${{ secrets.BEDROCK_AWS_ACCESS_KEY_ID }} + BEDROCK_AWS_ACCESS_KEY_ID: ${{ secrets.BEDROCK_AWS_ACCESS_KEY_ID }} \ No newline at end of file diff --git a/langchain-core/.gitignore b/langchain-core/.gitignore index 26f8fb1da8c8..247cd396048a 100644 --- a/langchain-core/.gitignore +++ b/langchain-core/.gitignore @@ -214,6 +214,10 @@ utils/types.cjs utils/types.js utils/types.d.ts utils/types.d.cts +utils/is_openai_tool.cjs +utils/is_openai_tool.js +utils/is_openai_tool.d.ts +utils/is_openai_tool.d.cts vectorstores.cjs vectorstores.js vectorstores.d.ts diff --git a/langchain-core/langchain.config.js b/langchain-core/langchain.config.js index cc1ba4060870..c1a20f6cad80 100644 --- a/langchain-core/langchain.config.js +++ b/langchain-core/langchain.config.js @@ -66,6 +66,7 @@ export const config = { "utils/testing": "utils/testing/index", "utils/tiktoken": "utils/tiktoken", "utils/types": "utils/types/index", + "utils/is_openai_tool": "utils/is_openai_tool", vectorstores: "vectorstores", }, tsConfigPath: resolve("./tsconfig.json"), diff --git a/langchain-core/package.json b/langchain-core/package.json index abc017dc35ce..0561ba93be66 100644 --- a/langchain-core/package.json +++ b/langchain-core/package.json @@ -581,6 +581,15 @@ "import": "./utils/types.js", "require": "./utils/types.cjs" }, + "./utils/is_openai_tool": { + "types": { + "import": "./utils/is_openai_tool.d.ts", + "require": "./utils/is_openai_tool.d.cts", + "default": "./utils/is_openai_tool.d.ts" + }, + "import": "./utils/is_openai_tool.js", + "require": "./utils/is_openai_tool.cjs" + }, "./vectorstores": { "types": { "import": "./vectorstores.d.ts", @@ -810,6 +819,10 @@ "utils/types.js", "utils/types.d.ts", "utils/types.d.cts", + "utils/is_openai_tool.cjs", + "utils/is_openai_tool.js", + "utils/is_openai_tool.d.ts", + "utils/is_openai_tool.d.cts", "vectorstores.cjs", "vectorstores.js", "vectorstores.d.ts", diff --git a/langchain-core/src/language_models/chat_models.ts b/langchain-core/src/language_models/chat_models.ts index d78bb04b2e0e..a44525fb7e86 100644 --- a/langchain-core/src/language_models/chat_models.ts +++ b/langchain-core/src/language_models/chat_models.ts @@ -1,3 +1,5 @@ +import { z } from "zod"; +import { zodToJsonSchema } from "zod-to-json-schema"; import { AIMessage, type BaseMessage, @@ -5,6 +7,7 @@ import { type BaseMessageLike, HumanMessage, coerceMessageLikeToMessage, + AIMessageChunk, } from "../messages/index.js"; import type { BasePromptValueInterface } from "../prompt_values.js"; import { @@ -17,6 +20,8 @@ import { } from "../outputs.js"; import { BaseLanguageModel, + StructuredOutputMethodOptions, + ToolDefinition, type BaseLanguageModelCallOptions, type BaseLanguageModelInput, type BaseLanguageModelParams, @@ -29,10 +34,16 @@ import { import type { RunnableConfig } from "../runnables/config.js"; import type { BaseCache } from "../caches.js"; import { StructuredToolInterface } from "../tools.js"; -import { Runnable } from "../runnables/base.js"; +import { + Runnable, + RunnableLambda, + RunnableSequence, +} from "../runnables/base.js"; import { isStreamEventsHandler } from "../tracers/event_stream.js"; import { isLogStreamHandler } from "../tracers/log_stream.js"; import { concat } from "../utils/stream.js"; +import { RunnablePassthrough } from "../runnables/passthrough.js"; +import { isZodSchema } from "../utils/types/is_zod_schema.js"; /** * Represents a serialized chat model. @@ -143,12 +154,16 @@ export abstract class BaseChatModel< * Bind tool-like objects to this chat model. * * @param tools A list of tool definitions to bind to this chat model. - * Can be a structured tool or an object matching the provider's - * specific tool schema. + * Can be a structured tool, an OpenAI formatted tool, or an object + * matching the provider's specific tool schema. * @param kwargs Any additional parameters to bind. */ bindTools?( - tools: (StructuredToolInterface | Record)[], + tools: ( + | StructuredToolInterface + | Record + | ToolDefinition + )[], kwargs?: Partial ): Runnable; @@ -714,6 +729,138 @@ export abstract class BaseChatModel< } return result.content; } + + withStructuredOutput< + // eslint-disable-next-line @typescript-eslint/no-explicit-any + RunOutput extends Record = Record + // eslint-disable-next-line @typescript-eslint/no-explicit-any + >( + outputSchema: + | z.ZodType + // eslint-disable-next-line @typescript-eslint/no-explicit-any + | Record, + config?: StructuredOutputMethodOptions + ): Runnable; + + withStructuredOutput< + // eslint-disable-next-line @typescript-eslint/no-explicit-any + RunOutput extends Record = Record + >( + outputSchema: + | z.ZodType + // eslint-disable-next-line @typescript-eslint/no-explicit-any + | Record, + config?: StructuredOutputMethodOptions + ): Runnable; + + withStructuredOutput< + // eslint-disable-next-line @typescript-eslint/no-explicit-any + RunOutput extends Record = Record + >( + outputSchema: + | z.ZodType + // eslint-disable-next-line @typescript-eslint/no-explicit-any + | Record, + config?: StructuredOutputMethodOptions + ): + | Runnable + | Runnable< + BaseLanguageModelInput, + { + raw: BaseMessage; + parsed: RunOutput; + } + > { + if (!("bindTools" in this) || typeof this.bindTools !== "function") { + throw new Error( + `Chat model must implement ".bindTools()" to use withStructuredOutput.` + ); + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const schema: z.ZodType | Record = outputSchema; + const name = config?.name; + const description = schema.description ?? "A function available to call."; + const method = config?.method; + const includeRaw = config?.includeRaw; + if (method === "jsonMode") { + throw new Error( + `Base withStructuredOutput implementation only supports "functionCalling" as a method.` + ); + } + + let functionName = name ?? "extract"; + let tools: ToolDefinition[]; + if (isZodSchema(schema)) { + tools = [ + { + type: "function", + function: { + name: functionName, + description, + parameters: zodToJsonSchema(schema), + }, + }, + ]; + } else { + if ("name" in schema) { + functionName = schema.name; + } + tools = [ + { + type: "function", + function: { + name: functionName, + description, + parameters: schema, + }, + }, + ]; + } + + const llm = this.bindTools(tools); + const outputParser = RunnableLambda.from( + (input: AIMessageChunk): RunOutput => { + if (!input.tool_calls || input.tool_calls.length === 0) { + throw new Error("No tool calls found in the response."); + } + const toolCall = input.tool_calls.find( + (tc) => tc.name === functionName + ); + if (!toolCall) { + throw new Error(`No tool call found with name ${functionName}.`); + } + return toolCall.args as RunOutput; + } + ); + + if (!includeRaw) { + return llm.pipe(outputParser).withConfig({ + runName: "StructuredOutput", + }) as Runnable; + } + + const parserAssign = RunnablePassthrough.assign({ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + parsed: (input: any, config) => outputParser.invoke(input.raw, config), + }); + const parserNone = RunnablePassthrough.assign({ + parsed: () => null, + }); + const parsedWithFallback = parserAssign.withFallbacks({ + fallbacks: [parserNone], + }); + return RunnableSequence.from< + BaseLanguageModelInput, + { raw: BaseMessage; parsed: RunOutput } + >([ + { + raw: llm, + }, + parsedWithFallback, + ]).withConfig({ + runName: "StructuredOutputRunnable", + }); + } } /** @@ -750,4 +897,4 @@ export abstract class SimpleChatModel< ], }; } -} +} \ No newline at end of file diff --git a/langchain-core/src/utils/is_openai_tool.ts b/langchain-core/src/utils/is_openai_tool.ts new file mode 100644 index 000000000000..40bb049506a9 --- /dev/null +++ b/langchain-core/src/utils/is_openai_tool.ts @@ -0,0 +1,17 @@ +import { ToolDefinition } from "../language_models/base.js"; + +export function isOpenAITool(tool: unknown): tool is ToolDefinition { + if (typeof tool !== "object" || !tool) return false; + if ( + "type" in tool && + tool.type === "function" && + "function" in tool && + typeof tool.function === "object" && + tool.function && + "name" in tool.function && + "parameters" in tool.function + ) { + return true; + } + return false; +} \ No newline at end of file From 592dba9c9b12f0708f10053d29be9a9eee0564d8 Mon Sep 17 00:00:00 2001 From: bracesproul Date: Wed, 12 Jun 2024 18:21:02 -0700 Subject: [PATCH 2/5] moved isOpenAITool to existing entrypoint --- langchain-core/.gitignore | 4 ---- langchain-core/langchain.config.js | 1 - langchain-core/package.json | 13 ------------- langchain-core/src/language_models/base.ts | 21 +++++++++++++++++++++ langchain-core/src/utils/is_openai_tool.ts | 17 ----------------- langchain-core/utils/is_openai_tool.cjs | 1 + langchain-core/utils/is_openai_tool.d.cts | 1 + langchain-core/utils/is_openai_tool.d.ts | 1 + langchain-core/utils/is_openai_tool.js | 1 + 9 files changed, 25 insertions(+), 35 deletions(-) delete mode 100644 langchain-core/src/utils/is_openai_tool.ts create mode 100644 langchain-core/utils/is_openai_tool.cjs create mode 100644 langchain-core/utils/is_openai_tool.d.cts create mode 100644 langchain-core/utils/is_openai_tool.d.ts create mode 100644 langchain-core/utils/is_openai_tool.js diff --git a/langchain-core/.gitignore b/langchain-core/.gitignore index 247cd396048a..26f8fb1da8c8 100644 --- a/langchain-core/.gitignore +++ b/langchain-core/.gitignore @@ -214,10 +214,6 @@ utils/types.cjs utils/types.js utils/types.d.ts utils/types.d.cts -utils/is_openai_tool.cjs -utils/is_openai_tool.js -utils/is_openai_tool.d.ts -utils/is_openai_tool.d.cts vectorstores.cjs vectorstores.js vectorstores.d.ts diff --git a/langchain-core/langchain.config.js b/langchain-core/langchain.config.js index c1a20f6cad80..cc1ba4060870 100644 --- a/langchain-core/langchain.config.js +++ b/langchain-core/langchain.config.js @@ -66,7 +66,6 @@ export const config = { "utils/testing": "utils/testing/index", "utils/tiktoken": "utils/tiktoken", "utils/types": "utils/types/index", - "utils/is_openai_tool": "utils/is_openai_tool", vectorstores: "vectorstores", }, tsConfigPath: resolve("./tsconfig.json"), diff --git a/langchain-core/package.json b/langchain-core/package.json index 0561ba93be66..abc017dc35ce 100644 --- a/langchain-core/package.json +++ b/langchain-core/package.json @@ -581,15 +581,6 @@ "import": "./utils/types.js", "require": "./utils/types.cjs" }, - "./utils/is_openai_tool": { - "types": { - "import": "./utils/is_openai_tool.d.ts", - "require": "./utils/is_openai_tool.d.cts", - "default": "./utils/is_openai_tool.d.ts" - }, - "import": "./utils/is_openai_tool.js", - "require": "./utils/is_openai_tool.cjs" - }, "./vectorstores": { "types": { "import": "./vectorstores.d.ts", @@ -819,10 +810,6 @@ "utils/types.js", "utils/types.d.ts", "utils/types.d.cts", - "utils/is_openai_tool.cjs", - "utils/is_openai_tool.js", - "utils/is_openai_tool.d.ts", - "utils/is_openai_tool.d.cts", "vectorstores.cjs", "vectorstores.js", "vectorstores.d.ts", diff --git a/langchain-core/src/language_models/base.ts b/langchain-core/src/language_models/base.ts index d9a0b4eead0e..b2770cd92df7 100644 --- a/langchain-core/src/language_models/base.ts +++ b/langchain-core/src/language_models/base.ts @@ -82,6 +82,27 @@ export const getModelContextSize = (modelName: string): number => { } }; +/** + * Whether or not the input matches the OpenAI tool definition. + * @param {unknown} tool The input to check. + * @returns {boolean} Whether the input is an OpenAI tool definition. + */ +export function isOpenAITool(tool: unknown): tool is ToolDefinition { + if (typeof tool !== "object" || !tool) return false; + if ( + "type" in tool && + tool.type === "function" && + "function" in tool && + typeof tool.function === "object" && + tool.function && + "name" in tool.function && + "parameters" in tool.function + ) { + return true; + } + return false; +} + interface CalculateMaxTokenProps { prompt: string; modelName: TiktokenModel; diff --git a/langchain-core/src/utils/is_openai_tool.ts b/langchain-core/src/utils/is_openai_tool.ts deleted file mode 100644 index 40bb049506a9..000000000000 --- a/langchain-core/src/utils/is_openai_tool.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { ToolDefinition } from "../language_models/base.js"; - -export function isOpenAITool(tool: unknown): tool is ToolDefinition { - if (typeof tool !== "object" || !tool) return false; - if ( - "type" in tool && - tool.type === "function" && - "function" in tool && - typeof tool.function === "object" && - tool.function && - "name" in tool.function && - "parameters" in tool.function - ) { - return true; - } - return false; -} \ No newline at end of file diff --git a/langchain-core/utils/is_openai_tool.cjs b/langchain-core/utils/is_openai_tool.cjs new file mode 100644 index 000000000000..2a65277be74c --- /dev/null +++ b/langchain-core/utils/is_openai_tool.cjs @@ -0,0 +1 @@ +module.exports = require('../dist/utils/is_openai_tool.cjs'); \ No newline at end of file diff --git a/langchain-core/utils/is_openai_tool.d.cts b/langchain-core/utils/is_openai_tool.d.cts new file mode 100644 index 000000000000..b0194f089619 --- /dev/null +++ b/langchain-core/utils/is_openai_tool.d.cts @@ -0,0 +1 @@ +export * from '../dist/utils/is_openai_tool.js' \ No newline at end of file diff --git a/langchain-core/utils/is_openai_tool.d.ts b/langchain-core/utils/is_openai_tool.d.ts new file mode 100644 index 000000000000..b0194f089619 --- /dev/null +++ b/langchain-core/utils/is_openai_tool.d.ts @@ -0,0 +1 @@ +export * from '../dist/utils/is_openai_tool.js' \ No newline at end of file diff --git a/langchain-core/utils/is_openai_tool.js b/langchain-core/utils/is_openai_tool.js new file mode 100644 index 000000000000..b0194f089619 --- /dev/null +++ b/langchain-core/utils/is_openai_tool.js @@ -0,0 +1 @@ +export * from '../dist/utils/is_openai_tool.js' \ No newline at end of file From e413a3c55f5832d26a26565663bc59b9497e2ab5 Mon Sep 17 00:00:00 2001 From: bracesproul Date: Thu, 13 Jun 2024 09:35:24 -0700 Subject: [PATCH 3/5] chore: lint files --- langchain-core/src/language_models/chat_models.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/langchain-core/src/language_models/chat_models.ts b/langchain-core/src/language_models/chat_models.ts index a44525fb7e86..f3b900427ba0 100644 --- a/langchain-core/src/language_models/chat_models.ts +++ b/langchain-core/src/language_models/chat_models.ts @@ -771,7 +771,7 @@ export abstract class BaseChatModel< parsed: RunOutput; } > { - if (!("bindTools" in this) || typeof this.bindTools !== "function") { + if (typeof this.bindTools !== "function") { throw new Error( `Chat model must implement ".bindTools()" to use withStructuredOutput.` ); @@ -897,4 +897,4 @@ export abstract class SimpleChatModel< ], }; } -} \ No newline at end of file +} From 626fc852293c23669e831a51c3aad3d4ea3ef134 Mon Sep 17 00:00:00 2001 From: bracesproul Date: Thu, 13 Jun 2024 09:52:24 -0700 Subject: [PATCH 4/5] cr --- .github/workflows/standard-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/standard-tests.yml b/.github/workflows/standard-tests.yml index cdcfd7bbf91c..50e73a31490c 100644 --- a/.github/workflows/standard-tests.yml +++ b/.github/workflows/standard-tests.yml @@ -13,7 +13,6 @@ jobs: package: [anthropic, cohere, google-genai, groq, mistralai] steps: - uses: actions/checkout@v4 - if: contains(needs.get-changed-files.outputs.changed_files, 'langchain-core/') || contains(needs.get-changed-files.outputs.changed_files, 'libs/langchain-${{ matrix.package }}/') - name: Use Node.js 18.x uses: actions/setup-node@v3 with: From 5dcb0b8cff3d99cbc45eeedc71f749156e48218d Mon Sep 17 00:00:00 2001 From: bracesproul Date: Fri, 21 Jun 2024 14:28:42 -0700 Subject: [PATCH 5/5] drop build artifacts --- langchain-core/utils/is_openai_tool.cjs | 1 - langchain-core/utils/is_openai_tool.d.cts | 1 - langchain-core/utils/is_openai_tool.d.ts | 1 - langchain-core/utils/is_openai_tool.js | 1 - 4 files changed, 4 deletions(-) delete mode 100644 langchain-core/utils/is_openai_tool.cjs delete mode 100644 langchain-core/utils/is_openai_tool.d.cts delete mode 100644 langchain-core/utils/is_openai_tool.d.ts delete mode 100644 langchain-core/utils/is_openai_tool.js diff --git a/langchain-core/utils/is_openai_tool.cjs b/langchain-core/utils/is_openai_tool.cjs deleted file mode 100644 index 2a65277be74c..000000000000 --- a/langchain-core/utils/is_openai_tool.cjs +++ /dev/null @@ -1 +0,0 @@ -module.exports = require('../dist/utils/is_openai_tool.cjs'); \ No newline at end of file diff --git a/langchain-core/utils/is_openai_tool.d.cts b/langchain-core/utils/is_openai_tool.d.cts deleted file mode 100644 index b0194f089619..000000000000 --- a/langchain-core/utils/is_openai_tool.d.cts +++ /dev/null @@ -1 +0,0 @@ -export * from '../dist/utils/is_openai_tool.js' \ No newline at end of file diff --git a/langchain-core/utils/is_openai_tool.d.ts b/langchain-core/utils/is_openai_tool.d.ts deleted file mode 100644 index b0194f089619..000000000000 --- a/langchain-core/utils/is_openai_tool.d.ts +++ /dev/null @@ -1 +0,0 @@ -export * from '../dist/utils/is_openai_tool.js' \ No newline at end of file diff --git a/langchain-core/utils/is_openai_tool.js b/langchain-core/utils/is_openai_tool.js deleted file mode 100644 index b0194f089619..000000000000 --- a/langchain-core/utils/is_openai_tool.js +++ /dev/null @@ -1 +0,0 @@ -export * from '../dist/utils/is_openai_tool.js' \ No newline at end of file