From 158744b017be439b653170737e435f77240ef0b3 Mon Sep 17 00:00:00 2001 From: jacoblee93 Date: Wed, 9 Oct 2024 15:25:32 -0700 Subject: [PATCH 1/5] Make OpenAI withStructuredOutput typing compatible with other models --- langchain-core/src/language_models/base.ts | 4 +-- .../src/language_models/chat_models.ts | 6 +++- .../chat_models/tests/chatbedrock.int.test.ts | 29 +++++++++++++++++++ libs/langchain-openai/src/chat_models.ts | 29 ++----------------- 4 files changed, 39 insertions(+), 29 deletions(-) diff --git a/langchain-core/src/language_models/base.ts b/langchain-core/src/language_models/base.ts index 4f1233426724..b7598cd51bfb 100644 --- a/langchain-core/src/language_models/base.ts +++ b/langchain-core/src/language_models/base.ts @@ -258,10 +258,11 @@ export type BaseLanguageModelInput = export type StructuredOutputType = z.infer>; export type StructuredOutputMethodOptions = - { + Record & { name?: string; method?: "functionCalling" | "jsonMode" | "jsonSchema" | string; includeRaw?: IncludeRaw; + strict?: boolean; }; /** @deprecated Use StructuredOutputMethodOptions instead */ @@ -514,7 +515,6 @@ export abstract class BaseLanguageModel< withStructuredOutput?< // eslint-disable-next-line @typescript-eslint/no-explicit-any RunOutput extends Record = Record - // eslint-disable-next-line @typescript-eslint/no-explicit-any >( schema: | z.ZodType diff --git a/langchain-core/src/language_models/chat_models.ts b/langchain-core/src/language_models/chat_models.ts index 43d820ed338f..c79bc00acbe1 100644 --- a/langchain-core/src/language_models/chat_models.ts +++ b/langchain-core/src/language_models/chat_models.ts @@ -797,7 +797,6 @@ export abstract class BaseChatModel< withStructuredOutput< // eslint-disable-next-line @typescript-eslint/no-explicit-any RunOutput extends Record = Record - // eslint-disable-next-line @typescript-eslint/no-explicit-any >( outputSchema: | z.ZodType @@ -840,6 +839,11 @@ export abstract class BaseChatModel< `Chat model must implement ".bindTools()" to use withStructuredOutput.` ); } + if (config?.strict) { + throw new Error( + `"strict" mode is not supported for this model by default.` + ); + } // eslint-disable-next-line @typescript-eslint/no-explicit-any const schema: z.ZodType | Record = outputSchema; const name = config?.name; diff --git a/libs/langchain-community/src/chat_models/tests/chatbedrock.int.test.ts b/libs/langchain-community/src/chat_models/tests/chatbedrock.int.test.ts index 926a47e0fd2b..dea7f739e350 100644 --- a/libs/langchain-community/src/chat_models/tests/chatbedrock.int.test.ts +++ b/libs/langchain-community/src/chat_models/tests/chatbedrock.int.test.ts @@ -11,6 +11,7 @@ import { z } from "zod"; import { zodToJsonSchema } from "zod-to-json-schema"; import { BedrockChat as BedrockChatWeb } from "../bedrock/web.js"; import { TavilySearchResults } from "../../tools/tavily_search.js"; +import { ChatOpenAI } from "@langchain/openai"; void testChatModel( "Test Bedrock chat model Generating search queries: Command-r", @@ -531,3 +532,31 @@ test("Streaming tool calls with Anthropic", async () => { expect(finalChunk?.tool_calls?.[0].name).toBe("weather_tool"); expect(finalChunk?.tool_calls?.[0].args?.city).toBeDefined(); }); + +test("withStructuredOutput result should be compatible with OpenAI typing", async () => { + const testSchema = z.object({ + thinking_process: z + .string() + .describe( + "Think before generating variants and put your reasoning here." + ), + variants: z + .array( + z.object({ + name: z.string(), + value: z.string(), + }) + ) + .describe("Variants of the input"), + }); + + const _prepareClient = () => { + if (Math.random() > 0.5) { + return new ChatOpenAI(); + } + + return new BedrockChatWeb(); + }; + + _prepareClient().withStructuredOutput(testSchema); +}); diff --git a/libs/langchain-openai/src/chat_models.ts b/libs/langchain-openai/src/chat_models.ts index 9d9d9a0da67d..8823baa42ae2 100644 --- a/libs/langchain-openai/src/chat_models.ts +++ b/libs/langchain-openai/src/chat_models.ts @@ -312,26 +312,6 @@ function _convertChatOpenAIToolTypeToOpenAITool( return _convertToOpenAITool(tool, fields); } -export interface ChatOpenAIStructuredOutputMethodOptions< - IncludeRaw extends boolean -> extends StructuredOutputMethodOptions { - /** - * strict: If `true` and `method` = "function_calling", model output is - * guaranteed to exactly match the schema. If `true`, the input schema - * will also be validated according to - * https://platform.openai.com/docs/guides/structured-outputs/supported-schemas. - * If `false`, input schema will not be validated and model output will not - * be validated. - * If `undefined`, `strict` argument will not be passed to the model. - * - * @version 0.2.6 - * @note Planned breaking change in version `0.3.0`: - * `strict` will default to `true` when `method` is - * "function_calling" as of version `0.3.0`. - */ - strict?: boolean; -} - export interface ChatOpenAICallOptions extends OpenAICallOptions, BaseFunctionCallOptions { @@ -1724,11 +1704,10 @@ export class ChatOpenAI< RunOutput extends Record = Record >( outputSchema: - | StructuredOutputMethodParams | z.ZodType // eslint-disable-next-line @typescript-eslint/no-explicit-any | Record, - config?: ChatOpenAIStructuredOutputMethodOptions + config?: StructuredOutputMethodOptions ): Runnable; withStructuredOutput< @@ -1736,11 +1715,10 @@ export class ChatOpenAI< RunOutput extends Record = Record >( outputSchema: - | StructuredOutputMethodParams | z.ZodType // eslint-disable-next-line @typescript-eslint/no-explicit-any | Record, - config?: ChatOpenAIStructuredOutputMethodOptions + config?: StructuredOutputMethodOptions ): Runnable; withStructuredOutput< @@ -1748,11 +1726,10 @@ export class ChatOpenAI< RunOutput extends Record = Record >( outputSchema: - | StructuredOutputMethodParams | z.ZodType // eslint-disable-next-line @typescript-eslint/no-explicit-any | Record, - config?: ChatOpenAIStructuredOutputMethodOptions + config?: StructuredOutputMethodOptions ): | Runnable | Runnable< From 03075a880d64544076a43f55373119e883922285 Mon Sep 17 00:00:00 2001 From: jacoblee93 Date: Mon, 11 Nov 2024 15:49:06 -0800 Subject: [PATCH 2/5] Fix --- langchain-core/src/language_models/base.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/langchain-core/src/language_models/base.ts b/langchain-core/src/language_models/base.ts index b7598cd51bfb..4177721339cd 100644 --- a/langchain-core/src/language_models/base.ts +++ b/langchain-core/src/language_models/base.ts @@ -258,10 +258,11 @@ export type BaseLanguageModelInput = export type StructuredOutputType = z.infer>; export type StructuredOutputMethodOptions = - Record & { + { name?: string; method?: "functionCalling" | "jsonMode" | "jsonSchema" | string; includeRaw?: IncludeRaw; + /** Whether to use strict mode. Currently only supported by OpenAI models. */ strict?: boolean; }; From 2f217950afe85e8db5bb95d79cdac3d22ab6c3d2 Mon Sep 17 00:00:00 2001 From: jacoblee93 Date: Mon, 11 Nov 2024 15:57:16 -0800 Subject: [PATCH 3/5] Revert --- libs/langchain-openai/src/chat_models.ts | 26 +++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/libs/langchain-openai/src/chat_models.ts b/libs/langchain-openai/src/chat_models.ts index 92737270270d..bd77a4896080 100644 --- a/libs/langchain-openai/src/chat_models.ts +++ b/libs/langchain-openai/src/chat_models.ts @@ -343,6 +343,26 @@ function _convertChatOpenAIToolTypeToOpenAITool( return _convertToOpenAITool(tool, fields); } +export interface ChatOpenAIStructuredOutputMethodOptions< + IncludeRaw extends boolean +> extends StructuredOutputMethodOptions { + /** + * strict: If `true` and `method` = "function_calling", model output is + * guaranteed to exactly match the schema. If `true`, the input schema + * will also be validated according to + * https://platform.openai.com/docs/guides/structured-outputs/supported-schemas. + * If `false`, input schema will not be validated and model output will not + * be validated. + * If `undefined`, `strict` argument will not be passed to the model. + * + * @version 0.2.6 + * @note Planned breaking change in version `0.3.0`: + * `strict` will default to `true` when `method` is + * "function_calling" as of version `0.3.0`. + */ + strict?: boolean; +} + export interface ChatOpenAICallOptions extends OpenAICallOptions, BaseFunctionCallOptions { @@ -1923,7 +1943,7 @@ export class ChatOpenAI< | z.ZodType // eslint-disable-next-line @typescript-eslint/no-explicit-any | Record, - config?: StructuredOutputMethodOptions + config?: ChatOpenAIStructuredOutputMethodOptions ): Runnable; withStructuredOutput< @@ -1934,7 +1954,7 @@ export class ChatOpenAI< | z.ZodType // eslint-disable-next-line @typescript-eslint/no-explicit-any | Record, - config?: StructuredOutputMethodOptions + config?: ChatOpenAIStructuredOutputMethodOptions ): Runnable; withStructuredOutput< @@ -1945,7 +1965,7 @@ export class ChatOpenAI< | z.ZodType // eslint-disable-next-line @typescript-eslint/no-explicit-any | Record, - config?: StructuredOutputMethodOptions + config?: ChatOpenAIStructuredOutputMethodOptions ): | Runnable | Runnable< From f4e05fa1d16f87b06883de6535ffe5e640d6d400 Mon Sep 17 00:00:00 2001 From: jacoblee93 Date: Mon, 11 Nov 2024 15:58:22 -0800 Subject: [PATCH 4/5] Fix --- libs/langchain-openai/src/chat_models.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/langchain-openai/src/chat_models.ts b/libs/langchain-openai/src/chat_models.ts index bd77a4896080..1db33c8728ab 100644 --- a/libs/langchain-openai/src/chat_models.ts +++ b/libs/langchain-openai/src/chat_models.ts @@ -343,6 +343,7 @@ function _convertChatOpenAIToolTypeToOpenAITool( return _convertToOpenAITool(tool, fields); } +// TODO: Use the base structured output options param in next breaking release. export interface ChatOpenAIStructuredOutputMethodOptions< IncludeRaw extends boolean > extends StructuredOutputMethodOptions { From da5e59c86730ea6a84db46bbd4829ee62e1b0198 Mon Sep 17 00:00:00 2001 From: jacoblee93 Date: Mon, 11 Nov 2024 16:15:30 -0800 Subject: [PATCH 5/5] Fix lint --- .../src/chat_models/tests/chatbedrock.int.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/langchain-community/src/chat_models/tests/chatbedrock.int.test.ts b/libs/langchain-community/src/chat_models/tests/chatbedrock.int.test.ts index dea7f739e350..ebce2edee869 100644 --- a/libs/langchain-community/src/chat_models/tests/chatbedrock.int.test.ts +++ b/libs/langchain-community/src/chat_models/tests/chatbedrock.int.test.ts @@ -9,9 +9,9 @@ import { ChatPromptTemplate } from "@langchain/core/prompts"; import { concat } from "@langchain/core/utils/stream"; import { z } from "zod"; import { zodToJsonSchema } from "zod-to-json-schema"; +import { ChatOpenAI } from "@langchain/openai"; import { BedrockChat as BedrockChatWeb } from "../bedrock/web.js"; import { TavilySearchResults } from "../../tools/tavily_search.js"; -import { ChatOpenAI } from "@langchain/openai"; void testChatModel( "Test Bedrock chat model Generating search queries: Command-r",