From a7bc9168449c3ed184b42c0a50536178c2bb6f71 Mon Sep 17 00:00:00 2001 From: Allen Firstenberg Date: Thu, 19 Dec 2024 17:25:47 -0500 Subject: [PATCH 01/13] fix(google-common, google-vertexai): Invalid URL for embeddings model (#7406) Co-authored-by: jacoblee93 --- .../langchain-google-common/src/embeddings.ts | 5 ++ .../src/tests/embeddings.int.test.ts | 47 ++++++++++++------- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/libs/langchain-google-common/src/embeddings.ts b/libs/langchain-google-common/src/embeddings.ts index d1e2549c631a..d3b7b8a2e2ab 100644 --- a/libs/langchain-google-common/src/embeddings.ts +++ b/libs/langchain-google-common/src/embeddings.ts @@ -38,6 +38,11 @@ class EmbeddingsConnection< return "predict"; } + get modelPublisher(): string { + // All the embedding models are currently published by "google" + return "google"; + } + async formatData( input: GoogleEmbeddingsInstance[], parameters: GoogleAIModelRequestParams diff --git a/libs/langchain-google-vertexai/src/tests/embeddings.int.test.ts b/libs/langchain-google-vertexai/src/tests/embeddings.int.test.ts index 5bf57a82b24b..72634c669a2e 100644 --- a/libs/langchain-google-vertexai/src/tests/embeddings.int.test.ts +++ b/libs/langchain-google-vertexai/src/tests/embeddings.int.test.ts @@ -9,21 +9,32 @@ test("Test VertexAIEmbeddings.embedQuery", async () => { expect(typeof res[0]).toBe("number"); }); -test("Test VertexAIEmbeddings.embedDocuments", async () => { - const embeddings = new VertexAIEmbeddings({ - model: "text-embedding-004", - }); - const res = await embeddings.embedDocuments([ - "Hello world", - "Bye bye", - "we need", - "at least", - "six documents", - "to test pagination", - ]); - // console.log(res); - expect(res).toHaveLength(6); - res.forEach((r) => { - expect(typeof r[0]).toBe("number"); - }); -}); +const testModelsLocations = [ + ["text-embedding-005", "us-central1"], + ["text-multilingual-embedding-002", "us-central1"], + ["text-embedding-005", "europe-west9"], + ["text-multilingual-embedding-002", "europe-west9"], +]; + +test.each(testModelsLocations)( + "VertexAIEmbeddings.embedDocuments %s %s", + async (model, location) => { + const embeddings = new VertexAIEmbeddings({ + model, + location, + }); + const res = await embeddings.embedDocuments([ + "Hello world", + "Bye bye", + "we need", + "at least", + "six documents", + "to test pagination", + ]); + // console.log(res); + expect(res).toHaveLength(6); + res.forEach((r) => { + expect(typeof r[0]).toBe("number"); + }); + } +); From 82cecaee82f0ef42650e6201410b7b4cceb16115 Mon Sep 17 00:00:00 2001 From: Kashif Minhaj Date: Fri, 20 Dec 2024 03:57:38 +0530 Subject: [PATCH 02/13] fix(google-common): add tool_use param when using structured outputs (#7400) --- libs/langchain-google-common/src/chat_models.ts | 1 + libs/langchain-google-common/src/utils/gemini.ts | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/libs/langchain-google-common/src/chat_models.ts b/libs/langchain-google-common/src/chat_models.ts index 1de5280fbe72..7d476b94cf2c 100644 --- a/libs/langchain-google-common/src/chat_models.ts +++ b/libs/langchain-google-common/src/chat_models.ts @@ -469,6 +469,7 @@ export abstract class ChatGoogleBase } const llm = this.bind({ tools, + tool_choice: functionName, }); if (!includeRaw) { diff --git a/libs/langchain-google-common/src/utils/gemini.ts b/libs/langchain-google-common/src/utils/gemini.ts index 48bd41fb5c2f..23c46f3783db 100644 --- a/libs/langchain-google-common/src/utils/gemini.ts +++ b/libs/langchain-google-common/src/utils/gemini.ts @@ -1047,10 +1047,20 @@ export function getGeminiAPI(config?: GeminiAPIConfig): GoogleAIAPI { return undefined; } + if (["auto", "any", "none"].includes(parameters.tool_choice)) { + return { + functionCallingConfig: { + mode: parameters.tool_choice as "auto" | "any" | "none", + allowedFunctionNames: parameters.allowed_function_names, + }, + }; + } + + // force tool choice to be a single function name in case of structured output return { functionCallingConfig: { - mode: parameters.tool_choice as "auto" | "any" | "none", - allowedFunctionNames: parameters.allowed_function_names, + mode: "any", + allowedFunctionNames: [parameters.tool_choice], }, }; } From 9d6b434e333d654cfe03f1bf3c3abc86b649cd5b Mon Sep 17 00:00:00 2001 From: oleg Date: Thu, 19 Dec 2024 23:28:03 +0100 Subject: [PATCH 03/13] fix(google-genai): Update multimodal model check to include gemini-2 support (#7389) Co-authored-by: jacoblee93 --- libs/langchain-google-genai/src/chat_models.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libs/langchain-google-genai/src/chat_models.ts b/libs/langchain-google-genai/src/chat_models.ts index 93f5dfa9c26f..fd09579bff09 100644 --- a/libs/langchain-google-genai/src/chat_models.ts +++ b/libs/langchain-google-genai/src/chat_models.ts @@ -580,7 +580,11 @@ export class ChatGoogleGenerativeAI private client: GenerativeModel; get _isMultimodalModel() { - return this.model.includes("vision") || this.model.startsWith("gemini-1.5"); + return ( + this.model.includes("vision") || + this.model.startsWith("gemini-1.5") || + this.model.startsWith("gemini-2") + ); } constructor(fields?: GoogleGenerativeAIChatInput) { From 6f6a25294185ed59b0195090268cd9ed468a4a08 Mon Sep 17 00:00:00 2001 From: Jacob Lee Date: Thu, 19 Dec 2024 14:32:17 -0800 Subject: [PATCH 04/13] release(google-genai): 0.1.6 (#7409) --- libs/langchain-google-genai/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/langchain-google-genai/package.json b/libs/langchain-google-genai/package.json index 5bf48183e567..47d4561964e4 100644 --- a/libs/langchain-google-genai/package.json +++ b/libs/langchain-google-genai/package.json @@ -1,6 +1,6 @@ { "name": "@langchain/google-genai", - "version": "0.1.5", + "version": "0.1.6", "description": "Google Generative AI integration for LangChain.js", "type": "module", "engines": { From 61b6cbbbf9a37745d12e360208c193ff4ebc0ee0 Mon Sep 17 00:00:00 2001 From: Jacob Lee Date: Thu, 19 Dec 2024 14:39:16 -0800 Subject: [PATCH 05/13] release(google-vertex): 0.1.5 (#7410) --- libs/langchain-google-common/package.json | 2 +- libs/langchain-google-gauth/package.json | 4 ++-- libs/langchain-google-vertexai-web/package.json | 4 ++-- libs/langchain-google-vertexai/package.json | 4 ++-- libs/langchain-google-webauth/package.json | 4 ++-- yarn.lock | 14 +++++++------- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/libs/langchain-google-common/package.json b/libs/langchain-google-common/package.json index 3408ef93e4ef..f5d32045f91c 100644 --- a/libs/langchain-google-common/package.json +++ b/libs/langchain-google-common/package.json @@ -1,6 +1,6 @@ { "name": "@langchain/google-common", - "version": "0.1.4", + "version": "0.1.5", "description": "Core types and classes for Google services.", "type": "module", "engines": { diff --git a/libs/langchain-google-gauth/package.json b/libs/langchain-google-gauth/package.json index 6a971b220ef6..c8851626c9f8 100644 --- a/libs/langchain-google-gauth/package.json +++ b/libs/langchain-google-gauth/package.json @@ -1,6 +1,6 @@ { "name": "@langchain/google-gauth", - "version": "0.1.4", + "version": "0.1.5", "description": "Google auth based authentication support for Google services", "type": "module", "engines": { @@ -35,7 +35,7 @@ "author": "LangChain", "license": "MIT", "dependencies": { - "@langchain/google-common": "~0.1.4", + "@langchain/google-common": "~0.1.5", "google-auth-library": "^8.9.0" }, "peerDependencies": { diff --git a/libs/langchain-google-vertexai-web/package.json b/libs/langchain-google-vertexai-web/package.json index 8761e8b75975..5d52d4680fce 100644 --- a/libs/langchain-google-vertexai-web/package.json +++ b/libs/langchain-google-vertexai-web/package.json @@ -1,6 +1,6 @@ { "name": "@langchain/google-vertexai-web", - "version": "0.1.4", + "version": "0.1.5", "description": "LangChain.js support for Google Vertex AI Web", "type": "module", "engines": { @@ -32,7 +32,7 @@ "author": "LangChain", "license": "MIT", "dependencies": { - "@langchain/google-webauth": "~0.1.4" + "@langchain/google-webauth": "~0.1.5" }, "peerDependencies": { "@langchain/core": ">=0.2.21 <0.4.0" diff --git a/libs/langchain-google-vertexai/package.json b/libs/langchain-google-vertexai/package.json index 0c238e99bc12..aa7f52592fec 100644 --- a/libs/langchain-google-vertexai/package.json +++ b/libs/langchain-google-vertexai/package.json @@ -1,6 +1,6 @@ { "name": "@langchain/google-vertexai", - "version": "0.1.4", + "version": "0.1.5", "description": "LangChain.js support for Google Vertex AI", "type": "module", "engines": { @@ -32,7 +32,7 @@ "author": "LangChain", "license": "MIT", "dependencies": { - "@langchain/google-gauth": "~0.1.4" + "@langchain/google-gauth": "~0.1.5" }, "peerDependencies": { "@langchain/core": ">=0.2.21 <0.4.0" diff --git a/libs/langchain-google-webauth/package.json b/libs/langchain-google-webauth/package.json index f3bee58fb805..0a46b9221e31 100644 --- a/libs/langchain-google-webauth/package.json +++ b/libs/langchain-google-webauth/package.json @@ -1,6 +1,6 @@ { "name": "@langchain/google-webauth", - "version": "0.1.4", + "version": "0.1.5", "description": "Web-based authentication support for Google services", "type": "module", "engines": { @@ -32,7 +32,7 @@ "author": "LangChain", "license": "MIT", "dependencies": { - "@langchain/google-common": "~0.1.4", + "@langchain/google-common": "~0.1.5", "web-auth-library": "^1.0.3" }, "peerDependencies": { diff --git a/yarn.lock b/yarn.lock index c27afffe82eb..9e5a48455320 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12413,7 +12413,7 @@ __metadata: languageName: unknown linkType: soft -"@langchain/google-common@^0.1.0, @langchain/google-common@workspace:*, @langchain/google-common@workspace:libs/langchain-google-common, @langchain/google-common@~0.1.4": +"@langchain/google-common@^0.1.0, @langchain/google-common@workspace:*, @langchain/google-common@workspace:libs/langchain-google-common, @langchain/google-common@~0.1.5": version: 0.0.0-use.local resolution: "@langchain/google-common@workspace:libs/langchain-google-common" dependencies: @@ -12448,13 +12448,13 @@ __metadata: languageName: unknown linkType: soft -"@langchain/google-gauth@workspace:libs/langchain-google-gauth, @langchain/google-gauth@~0.1.4": +"@langchain/google-gauth@workspace:libs/langchain-google-gauth, @langchain/google-gauth@~0.1.5": version: 0.0.0-use.local resolution: "@langchain/google-gauth@workspace:libs/langchain-google-gauth" dependencies: "@jest/globals": ^29.5.0 "@langchain/core": "workspace:*" - "@langchain/google-common": ~0.1.4 + "@langchain/google-common": ~0.1.5 "@langchain/scripts": ">=0.1.0 <0.2.0" "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 @@ -12527,7 +12527,7 @@ __metadata: "@jest/globals": ^29.5.0 "@langchain/core": "workspace:*" "@langchain/google-common": ^0.1.0 - "@langchain/google-webauth": ~0.1.4 + "@langchain/google-webauth": ~0.1.5 "@langchain/scripts": ">=0.1.0 <0.2.0" "@langchain/standard-tests": 0.0.0 "@swc/core": ^1.3.90 @@ -12563,7 +12563,7 @@ __metadata: "@jest/globals": ^29.5.0 "@langchain/core": "workspace:*" "@langchain/google-common": ^0.1.0 - "@langchain/google-gauth": ~0.1.4 + "@langchain/google-gauth": ~0.1.5 "@langchain/scripts": ">=0.1.0 <0.2.0" "@langchain/standard-tests": 0.0.0 "@swc/core": ^1.3.90 @@ -12592,13 +12592,13 @@ __metadata: languageName: unknown linkType: soft -"@langchain/google-webauth@workspace:libs/langchain-google-webauth, @langchain/google-webauth@~0.1.4": +"@langchain/google-webauth@workspace:libs/langchain-google-webauth, @langchain/google-webauth@~0.1.5": version: 0.0.0-use.local resolution: "@langchain/google-webauth@workspace:libs/langchain-google-webauth" dependencies: "@jest/globals": ^29.5.0 "@langchain/core": "workspace:*" - "@langchain/google-common": ~0.1.4 + "@langchain/google-common": ~0.1.5 "@langchain/scripts": ">=0.1.0 <0.2.0" "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 From 1a4e0ca4c5a34f5498832857e7c09d8289fe04be Mon Sep 17 00:00:00 2001 From: Jacob Lee Date: Thu, 19 Dec 2024 17:23:08 -0800 Subject: [PATCH 06/13] fix(core): Fix runnable with fallbacks stream events bug (#7411) --- langchain-core/src/runnables/base.ts | 49 +++++++++++-------- .../tests/runnable_with_fallbacks.test.ts | 37 ++++++++++++++ 2 files changed, 65 insertions(+), 21 deletions(-) diff --git a/langchain-core/src/runnables/base.ts b/langchain-core/src/runnables/base.ts index d2ba10b6f6bd..09c8c7162ee9 100644 --- a/langchain-core/src/runnables/base.ts +++ b/langchain-core/src/runnables/base.ts @@ -2840,7 +2840,7 @@ export class RunnableWithFallbacks extends Runnable< options?: Partial ): Promise { const config = ensureConfig(options); - const callbackManager_ = await getCallbackManagerForConfig(options); + const callbackManager_ = await getCallbackManagerForConfig(config); const { runId, ...otherConfigFields } = config; const runManager = await callbackManager_?.handleChainStart( this.toJSON(), @@ -2851,27 +2851,33 @@ export class RunnableWithFallbacks extends Runnable< undefined, otherConfigFields?.runName ); - let firstError; - for (const runnable of this.runnables()) { - config?.signal?.throwIfAborted(); - try { - const output = await runnable.invoke( - input, - patchConfig(otherConfigFields, { callbacks: runManager?.getChild() }) - ); - await runManager?.handleChainEnd(_coerceToDict(output, "output")); - return output; - } catch (e) { + const childConfig = patchConfig(otherConfigFields, { + callbacks: runManager?.getChild(), + }); + const res = await AsyncLocalStorageProviderSingleton.runWithConfig( + childConfig, + async () => { + let firstError; + for (const runnable of this.runnables()) { + config?.signal?.throwIfAborted(); + try { + const output = await runnable.invoke(input, childConfig); + await runManager?.handleChainEnd(_coerceToDict(output, "output")); + return output; + } catch (e) { + if (firstError === undefined) { + firstError = e; + } + } + } if (firstError === undefined) { - firstError = e; + throw new Error("No error stored at end of fallback."); } + await runManager?.handleChainError(firstError); + throw firstError; } - } - if (firstError === undefined) { - throw new Error("No error stored at end of fallback."); - } - await runManager?.handleChainError(firstError); - throw firstError; + ); + return res; } async *_streamIterator( @@ -2879,7 +2885,7 @@ export class RunnableWithFallbacks extends Runnable< options?: Partial | undefined ): AsyncGenerator { const config = ensureConfig(options); - const callbackManager_ = await getCallbackManagerForConfig(options); + const callbackManager_ = await getCallbackManagerForConfig(config); const { runId, ...otherConfigFields } = config; const runManager = await callbackManager_?.handleChainStart( this.toJSON(), @@ -2898,7 +2904,8 @@ export class RunnableWithFallbacks extends Runnable< callbacks: runManager?.getChild(), }); try { - stream = await runnable.stream(input, childConfig); + const originalStream = await runnable.stream(input, childConfig); + stream = consumeAsyncIterableInContext(childConfig, originalStream); break; } catch (e) { if (firstError === undefined) { diff --git a/langchain-core/src/runnables/tests/runnable_with_fallbacks.test.ts b/langchain-core/src/runnables/tests/runnable_with_fallbacks.test.ts index 652116fd70ee..7cb5c5fe04c5 100644 --- a/langchain-core/src/runnables/tests/runnable_with_fallbacks.test.ts +++ b/langchain-core/src/runnables/tests/runnable_with_fallbacks.test.ts @@ -1,7 +1,11 @@ /* eslint-disable no-promise-executor-return */ /* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable no-process-env */ import { test, expect } from "@jest/globals"; +import { AsyncLocalStorage } from "node:async_hooks"; import { FakeLLM, FakeStreamingLLM } from "../../utils/testing/index.js"; +import { RunnableLambda } from "../base.js"; +import { AsyncLocalStorageProviderSingleton } from "../../singletons/index.js"; test("RunnableWithFallbacks", async () => { const llm = new FakeLLM({ @@ -55,3 +59,36 @@ test("RunnableWithFallbacks stream", async () => { expect(chunks.length).toBeGreaterThan(1); expect(chunks.join("")).toEqual("What up"); }); + +test("RunnableWithFallbacks stream events with local storage and callbacks added via env vars", async () => { + process.env.LANGCHAIN_VERBOSE = "true"; + AsyncLocalStorageProviderSingleton.initializeGlobalInstance( + new AsyncLocalStorage() + ); + const llm = new FakeStreamingLLM({ + thrownErrorString: "Bad error!", + }); + const llmWithFallbacks = llm.withFallbacks({ + fallbacks: [new FakeStreamingLLM({})], + }); + const runnable = RunnableLambda.from(async (input: any) => { + const res = await llmWithFallbacks.invoke(input); + const stream = await llmWithFallbacks.stream(input); + for await (const _ of stream) { + void _; + } + return res; + }); + const stream = await runnable.streamEvents("hi", { + version: "v2", + }); + const chunks = []; + for await (const chunk of stream) { + if (chunk.event === "on_llm_stream") { + chunks.push(chunk); + } + } + expect(chunks.length).toBeGreaterThan(1); + console.log(JSON.stringify(chunks, null, 2)); + expect(chunks.map((chunk) => chunk.data.chunk.text).join("")).toEqual("hihi"); +}); From a0f7d401ed9c9c2ecb24a97d0ed650e40f825846 Mon Sep 17 00:00:00 2001 From: Jacob Lee Date: Thu, 19 Dec 2024 17:26:21 -0800 Subject: [PATCH 07/13] release(core): 0.3.26 (#7412) --- langchain-core/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langchain-core/package.json b/langchain-core/package.json index ffb9c44d1fd8..32510ecc8c72 100644 --- a/langchain-core/package.json +++ b/langchain-core/package.json @@ -1,6 +1,6 @@ { "name": "@langchain/core", - "version": "0.3.25", + "version": "0.3.26", "description": "Core LangChain.js abstractions and schemas", "type": "module", "engines": { From 51bf619f8d45edcc39926a876c5e61b8fc6aad4a Mon Sep 17 00:00:00 2001 From: crisjy Date: Fri, 20 Dec 2024 09:33:20 +0800 Subject: [PATCH 08/13] fix(azure-cosmosdb): Add caches for Azure CosmosDB vCore [comments resolved] (#7307) Co-authored-by: jacoblee93 --- .../src/caches/caches_mongodb.ts | 178 ++++++++++++++++++ .../src/{caches.ts => caches/caches_nosql.ts} | 2 +- libs/langchain-azure-cosmosdb/src/index.ts | 3 +- .../tests/caches/caches_mongodb.int.test.ts | 136 +++++++++++++ .../src/tests/caches/caches_mongodb.test.ts | 72 +++++++ .../caches_nosql.int.test.ts} | 2 +- .../caches_nosql.test.ts} | 2 +- 7 files changed, 391 insertions(+), 4 deletions(-) create mode 100644 libs/langchain-azure-cosmosdb/src/caches/caches_mongodb.ts rename libs/langchain-azure-cosmosdb/src/{caches.ts => caches/caches_nosql.ts} (99%) create mode 100644 libs/langchain-azure-cosmosdb/src/tests/caches/caches_mongodb.int.test.ts create mode 100644 libs/langchain-azure-cosmosdb/src/tests/caches/caches_mongodb.test.ts rename libs/langchain-azure-cosmosdb/src/tests/{caches.int.test.ts => caches/caches_nosql.int.test.ts} (99%) rename libs/langchain-azure-cosmosdb/src/tests/{caches.test.ts => caches/caches_nosql.test.ts} (96%) diff --git a/libs/langchain-azure-cosmosdb/src/caches/caches_mongodb.ts b/libs/langchain-azure-cosmosdb/src/caches/caches_mongodb.ts new file mode 100644 index 000000000000..b33589a4095d --- /dev/null +++ b/libs/langchain-azure-cosmosdb/src/caches/caches_mongodb.ts @@ -0,0 +1,178 @@ +import { + BaseCache, + deserializeStoredGeneration, + getCacheKey, + serializeGeneration, +} from "@langchain/core/caches"; +import { Generation } from "@langchain/core/outputs"; +import { Document } from "@langchain/core/documents"; +import { EmbeddingsInterface } from "@langchain/core/embeddings"; +import { getEnvironmentVariable } from "@langchain/core/utils/env"; +import { MongoClient } from "mongodb"; +import { + AzureCosmosDBMongoDBConfig, + AzureCosmosDBMongoDBVectorStore, + AzureCosmosDBMongoDBSimilarityType, +} from "../azure_cosmosdb_mongodb.js"; + +/** + * Represents a Semantic Cache that uses CosmosDB MongoDB backend as the underlying + * storage system. + * + * @example + * ```typescript + * const embeddings = new OpenAIEmbeddings(); + * const cache = new AzureCosmosDBMongoDBSemanticCache(embeddings, { + * client?: MongoClient + * }); + * const model = new ChatOpenAI({cache}); + * + * // Invoke the model to perform an action + * const response = await model.invoke("Do something random!"); + * console.log(response); + * ``` + */ +export class AzureCosmosDBMongoDBSemanticCache extends BaseCache { + private embeddings: EmbeddingsInterface; + + private config: AzureCosmosDBMongoDBConfig; + + private similarityScoreThreshold: number; + + private cacheDict: { [key: string]: AzureCosmosDBMongoDBVectorStore } = {}; + + private readonly client: MongoClient | undefined; + + private vectorDistanceFunction: string; + + constructor( + embeddings: EmbeddingsInterface, + dbConfig: AzureCosmosDBMongoDBConfig, + similarityScoreThreshold: number = 0.6 + ) { + super(); + + const connectionString = + dbConfig.connectionString ?? + getEnvironmentVariable("AZURE_COSMOSDB_MONGODB_CONNECTION_STRING"); + + if (!dbConfig.client && !connectionString) { + throw new Error( + "AzureCosmosDBMongoDBSemanticCache client or connection string must be set." + ); + } + + if (!dbConfig.client) { + this.client = new MongoClient(connectionString!, { + appName: "langchainjs", + }); + } else { + this.client = dbConfig.client; + } + + this.config = { + ...dbConfig, + client: this.client, + collectionName: dbConfig.collectionName ?? "semanticCacheContainer", + }; + + this.similarityScoreThreshold = similarityScoreThreshold; + this.embeddings = embeddings; + this.vectorDistanceFunction = + dbConfig?.indexOptions?.similarity ?? + AzureCosmosDBMongoDBSimilarityType.COS; + } + + private getLlmCache(llmKey: string) { + const key = getCacheKey(llmKey); + if (!this.cacheDict[key]) { + this.cacheDict[key] = new AzureCosmosDBMongoDBVectorStore( + this.embeddings, + this.config + ); + } + return this.cacheDict[key]; + } + + /** + * Retrieves data from the cache. + * + * @param prompt The prompt for lookup. + * @param llmKey The LLM key used to construct the cache key. + * @returns An array of Generations if found, null otherwise. + */ + async lookup(prompt: string, llmKey: string): Promise { + const llmCache = this.getLlmCache(llmKey); + + const queryEmbedding = await this.embeddings.embedQuery(prompt); + const results = await llmCache.similaritySearchVectorWithScore( + queryEmbedding, + 1, + this.config.indexOptions?.indexType + ); + if (!results.length) return null; + + const generations = results + .flatMap(([document, score]) => { + const isSimilar = + (this.vectorDistanceFunction === + AzureCosmosDBMongoDBSimilarityType.L2 && + score <= this.similarityScoreThreshold) || + (this.vectorDistanceFunction !== + AzureCosmosDBMongoDBSimilarityType.L2 && + score >= this.similarityScoreThreshold); + + if (!isSimilar) return undefined; + + return document.metadata.return_value.map((gen: string) => + deserializeStoredGeneration(JSON.parse(gen)) + ); + }) + .filter((gen) => gen !== undefined); + + return generations.length > 0 ? generations : null; + } + + /** + * Updates the cache with new data. + * + * @param prompt The prompt for update. + * @param llmKey The LLM key used to construct the cache key. + * @param value The value to be stored in the cache. + */ + public async update( + prompt: string, + llmKey: string, + returnValue: Generation[] + ): Promise { + const serializedGenerations = returnValue.map((generation) => + JSON.stringify(serializeGeneration(generation)) + ); + + const llmCache = this.getLlmCache(llmKey); + + const metadata = { + llm_string: llmKey, + prompt, + return_value: serializedGenerations, + }; + + const doc = new Document({ + pageContent: prompt, + metadata, + }); + + await llmCache.addDocuments([doc]); + } + + /** + * deletes the semantic cache for a given llmKey + * @param llmKey + */ + public async clear(llmKey: string) { + const key = getCacheKey(llmKey); + if (this.cacheDict[key]) { + await this.cacheDict[key].delete(); + } + } +} diff --git a/libs/langchain-azure-cosmosdb/src/caches.ts b/libs/langchain-azure-cosmosdb/src/caches/caches_nosql.ts similarity index 99% rename from libs/langchain-azure-cosmosdb/src/caches.ts rename to libs/langchain-azure-cosmosdb/src/caches/caches_nosql.ts index da7619c5ff96..e110f848702b 100644 --- a/libs/langchain-azure-cosmosdb/src/caches.ts +++ b/libs/langchain-azure-cosmosdb/src/caches/caches_nosql.ts @@ -13,7 +13,7 @@ import { getEnvironmentVariable } from "@langchain/core/utils/env"; import { AzureCosmosDBNoSQLConfig, AzureCosmosDBNoSQLVectorStore, -} from "./azure_cosmosdb_nosql.js"; +} from "../azure_cosmosdb_nosql.js"; const USER_AGENT_SUFFIX = "langchainjs-cdbnosql-semanticcache-javascript"; const DEFAULT_CONTAINER_NAME = "semanticCacheContainer"; diff --git a/libs/langchain-azure-cosmosdb/src/index.ts b/libs/langchain-azure-cosmosdb/src/index.ts index 883710c842ff..2778d2c55417 100644 --- a/libs/langchain-azure-cosmosdb/src/index.ts +++ b/libs/langchain-azure-cosmosdb/src/index.ts @@ -1,5 +1,6 @@ export * from "./azure_cosmosdb_mongodb.js"; export * from "./azure_cosmosdb_nosql.js"; -export * from "./caches.js"; +export * from "./caches/caches_nosql.js"; +export * from "./caches/caches_mongodb.js"; export * from "./chat_histories/nosql.js"; export * from "./chat_histories/mongodb.js"; diff --git a/libs/langchain-azure-cosmosdb/src/tests/caches/caches_mongodb.int.test.ts b/libs/langchain-azure-cosmosdb/src/tests/caches/caches_mongodb.int.test.ts new file mode 100644 index 000000000000..7902bdc06e79 --- /dev/null +++ b/libs/langchain-azure-cosmosdb/src/tests/caches/caches_mongodb.int.test.ts @@ -0,0 +1,136 @@ +/* eslint-disable no-nested-ternary */ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable no-process-env */ +import { ChatOpenAI, OpenAIEmbeddings } from "@langchain/openai"; +import { MongoClient } from "mongodb"; +import { AzureCosmosDBMongoDBSemanticCache } from "../../caches/caches_mongodb.js"; +import { + AzureCosmosDBMongoDBIndexOptions, + AzureCosmosDBMongoDBSimilarityType, +} from "../../azure_cosmosdb_mongodb.js"; + +const DATABASE_NAME = "langchain"; +const COLLECTION_NAME = "test"; + +async function initializeCache( + indexType: any, + distanceFunction: any, + similarityThreshold: number = 0.6 +): Promise { + const embeddingModel = new OpenAIEmbeddings(); + const testEmbedding = await embeddingModel.embedDocuments(["sample text"]); + const dimension = testEmbedding[0].length; + + const indexOptions: AzureCosmosDBMongoDBIndexOptions = { + indexType, + // eslint-disable-next-line no-nested-ternary + similarity: + distanceFunction === "cosine" + ? AzureCosmosDBMongoDBSimilarityType.COS + : distanceFunction === "euclidean" + ? AzureCosmosDBMongoDBSimilarityType.L2 + : AzureCosmosDBMongoDBSimilarityType.IP, + dimensions: dimension, + }; + + let cache: AzureCosmosDBMongoDBSemanticCache; + + const connectionString = process.env.AZURE_COSMOSDB_MONGODB_CONNECTION_STRING; + if (connectionString) { + cache = new AzureCosmosDBMongoDBSemanticCache( + embeddingModel, + { + databaseName: DATABASE_NAME, + collectionName: COLLECTION_NAME, + connectionString, + indexOptions, + }, + similarityThreshold + ); + } else { + throw new Error( + "Please set the environment variable AZURE_COSMOSDB_MONGODB_CONNECTION_STRING" + ); + } + + return cache; +} + +describe("AzureCosmosDBMongoDBSemanticCache", () => { + beforeEach(async () => { + const connectionString = + process.env.AZURE_COSMOSDB_MONGODB_CONNECTION_STRING; + const client = new MongoClient(connectionString!); + + try { + await client.db(DATABASE_NAME).collection(COLLECTION_NAME).drop(); + } catch (error) { + throw new Error("Please set collection name here"); + } + }); + + it("should store and retrieve cache using cosine similarity with ivf index", async () => { + const cache = await initializeCache("ivf", "cosine"); + const model = new ChatOpenAI({ cache }); + const llmString = JSON.stringify(model._identifyingParams); + await cache.update("foo", llmString, [{ text: "fizz" }]); + + let cacheOutput = await cache.lookup("foo", llmString); + expect(cacheOutput).toEqual([{ text: "fizz" }]); + + cacheOutput = await cache.lookup("bar", llmString); + expect(cacheOutput).toEqual(null); + + await cache.clear(llmString); + }); + + it("should store and retrieve cache using euclidean similarity with hnsw index", async () => { + const cache = await initializeCache("hnsw", "euclidean"); + const model = new ChatOpenAI({ cache }); + const llmString = JSON.stringify(model._identifyingParams); + await cache.update("foo", llmString, [{ text: "fizz" }]); + + let cacheOutput = await cache.lookup("foo", llmString); + expect(cacheOutput).toEqual([{ text: "fizz" }]); + + cacheOutput = await cache.lookup("bar", llmString); + expect(cacheOutput).toEqual(null); + + await cache.clear(llmString); + }); + + it("should return null if similarity score is below threshold (cosine similarity with ivf index)", async () => { + const cache = await initializeCache("ivf", "cosine", 0.8); + const model = new ChatOpenAI({ cache }); + const llmString = JSON.stringify(model._identifyingParams); + await cache.update("foo", llmString, [{ text: "fizz" }]); + + const cacheOutput = await cache.lookup("foo", llmString); + expect(cacheOutput).toEqual([{ text: "fizz" }]); + + const resultBelowThreshold = await cache.lookup("bar", llmString); + expect(resultBelowThreshold).toEqual(null); + + await cache.clear(llmString); + }); + + it("should handle a variety of cache updates and lookups", async () => { + const cache = await initializeCache("ivf", "cosine", 0.7); + const model = new ChatOpenAI({ cache }); + const llmString = JSON.stringify(model._identifyingParams); + + await cache.update("test1", llmString, [{ text: "response 1" }]); + await cache.update("test2", llmString, [{ text: "response 2" }]); + + let cacheOutput = await cache.lookup("test1", llmString); + expect(cacheOutput).toEqual([{ text: "response 1" }]); + + cacheOutput = await cache.lookup("test2", llmString); + expect(cacheOutput).toEqual([{ text: "response 2" }]); + + cacheOutput = await cache.lookup("test3", llmString); + expect(cacheOutput).toEqual(null); + + await cache.clear(llmString); + }); +}); diff --git a/libs/langchain-azure-cosmosdb/src/tests/caches/caches_mongodb.test.ts b/libs/langchain-azure-cosmosdb/src/tests/caches/caches_mongodb.test.ts new file mode 100644 index 000000000000..4bd9cf0ce996 --- /dev/null +++ b/libs/langchain-azure-cosmosdb/src/tests/caches/caches_mongodb.test.ts @@ -0,0 +1,72 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +// eslint-disable-next-line import/no-extraneous-dependencies +import { jest } from "@jest/globals"; +import { FakeEmbeddings, FakeLLM } from "@langchain/core/utils/testing"; +import { Document } from "@langchain/core/documents"; +import { MongoClient } from "mongodb"; +import { AzureCosmosDBMongoDBSemanticCache } from "../../index.js"; + +const createMockClient = () => ({ + db: jest.fn().mockReturnValue({ + collectionName: "documents", + collection: jest.fn().mockReturnValue({ + listIndexes: jest.fn().mockReturnValue({ + toArray: jest.fn().mockReturnValue([ + { + name: "vectorSearchIndex", + }, + ]), + }), + findOne: jest.fn().mockReturnValue({ + metadata: { + return_value: ['{"text": "fizz"}'], + }, + similarityScore: 0.8, + }), + insertMany: jest.fn().mockImplementation((docs: any) => ({ + insertedIds: docs.map((_: any, i: any) => `id${i}`), + })), + aggregate: jest.fn().mockReturnValue({ + map: jest.fn().mockReturnValue({ + toArray: jest.fn().mockReturnValue([ + [ + new Document({ + pageContent: "test", + metadata: { return_value: ['{"text": "fizz"}'] }, + }), + 0.8, + ], + ]), + }), + }), + }), + command: jest.fn(), + }), + connect: jest.fn(), + close: jest.fn(), +}); + +describe("AzureCosmosDBMongoDBSemanticCache", () => { + it("should store, retrieve, and clear cache in MongoDB", async () => { + const mockClient = createMockClient() as any; + const embeddings = new FakeEmbeddings(); + const cache = new AzureCosmosDBMongoDBSemanticCache( + embeddings, + { + client: mockClient as MongoClient, + }, + 0.8 + ); + + expect(cache).toBeDefined(); + + const llm = new FakeLLM({}); + const llmString = JSON.stringify(llm._identifyingParams()); + + await cache.update("foo", llmString, [{ text: "fizz" }]); + expect(mockClient.db().collection().insertMany).toHaveBeenCalled(); + + const result = await cache.lookup("foo", llmString); + expect(result).toEqual([{ text: "fizz" }]); + }); +}); diff --git a/libs/langchain-azure-cosmosdb/src/tests/caches.int.test.ts b/libs/langchain-azure-cosmosdb/src/tests/caches/caches_nosql.int.test.ts similarity index 99% rename from libs/langchain-azure-cosmosdb/src/tests/caches.int.test.ts rename to libs/langchain-azure-cosmosdb/src/tests/caches/caches_nosql.int.test.ts index c7acb92f7c86..4c301469bbcf 100644 --- a/libs/langchain-azure-cosmosdb/src/tests/caches.int.test.ts +++ b/libs/langchain-azure-cosmosdb/src/tests/caches/caches_nosql.int.test.ts @@ -9,7 +9,7 @@ import { } from "@azure/cosmos"; import { DefaultAzureCredential } from "@azure/identity"; import { ChatOpenAI, OpenAIEmbeddings } from "@langchain/openai"; -import { AzureCosmosDBNoSQLSemanticCache } from "../caches.js"; +import { AzureCosmosDBNoSQLSemanticCache } from "../../caches/caches_nosql.js"; const DATABASE_NAME = "langchainTestCacheDB"; const CONTAINER_NAME = "testContainer"; diff --git a/libs/langchain-azure-cosmosdb/src/tests/caches.test.ts b/libs/langchain-azure-cosmosdb/src/tests/caches/caches_nosql.test.ts similarity index 96% rename from libs/langchain-azure-cosmosdb/src/tests/caches.test.ts rename to libs/langchain-azure-cosmosdb/src/tests/caches/caches_nosql.test.ts index 9de3f507acc0..3a7a253f22bc 100644 --- a/libs/langchain-azure-cosmosdb/src/tests/caches.test.ts +++ b/libs/langchain-azure-cosmosdb/src/tests/caches/caches_nosql.test.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ import { jest } from "@jest/globals"; import { FakeEmbeddings, FakeLLM } from "@langchain/core/utils/testing"; -import { AzureCosmosDBNoSQLSemanticCache } from "../index.js"; +import { AzureCosmosDBNoSQLSemanticCache } from "../../index.js"; // Create the mock Cosmos DB client const createMockClient = () => { From 5afc890ab8ccd1c5738b2235c70c0779502ceed9 Mon Sep 17 00:00:00 2001 From: Jacob Lee Date: Thu, 19 Dec 2024 17:35:56 -0800 Subject: [PATCH 09/13] release(azure-cosmosdb): 0.2.6 (#7413) --- libs/langchain-azure-cosmosdb/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/langchain-azure-cosmosdb/package.json b/libs/langchain-azure-cosmosdb/package.json index d57ef57f7b96..d958472cf933 100644 --- a/libs/langchain-azure-cosmosdb/package.json +++ b/libs/langchain-azure-cosmosdb/package.json @@ -1,6 +1,6 @@ { "name": "@langchain/azure-cosmosdb", - "version": "0.2.5", + "version": "0.2.6", "description": "Azure CosmosDB integration for LangChain.js", "type": "module", "engines": { From 62bcabda1328322520594ddf3d0503c6fe2689b0 Mon Sep 17 00:00:00 2001 From: Ranjeet Baraik <84462743+anadi45@users.noreply.github.com> Date: Fri, 20 Dec 2024 07:06:11 +0530 Subject: [PATCH 10/13] fix(openai): include model_name in response_metadata (#7403) --- libs/langchain-openai/src/chat_models.ts | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/libs/langchain-openai/src/chat_models.ts b/libs/langchain-openai/src/chat_models.ts index f2151c8e0a50..0906d2848655 100644 --- a/libs/langchain-openai/src/chat_models.ts +++ b/libs/langchain-openai/src/chat_models.ts @@ -173,13 +173,15 @@ function openAIResponseToChatMessage( if (includeRawResponse !== undefined) { additional_kwargs.__raw_response = rawResponse; } - let response_metadata: Record | undefined; - if (rawResponse.system_fingerprint) { - response_metadata = { - usage: { ...rawResponse.usage }, - system_fingerprint: rawResponse.system_fingerprint, - }; - } + const response_metadata: Record | undefined = { + model_name: rawResponse.model, + ...(rawResponse.system_fingerprint + ? { + usage: { ...rawResponse.usage }, + system_fingerprint: rawResponse.system_fingerprint, + } + : {}), + }; if (message.audio) { additional_kwargs.audio = message.audio; @@ -1443,6 +1445,7 @@ export class ChatOpenAI< // Only include system fingerprint in the last chunk for now // to avoid concatenation issues generationInfo.system_fingerprint = data.system_fingerprint; + generationInfo.model_name = data.model; } if (this.logprobs) { generationInfo.logprobs = choice.logprobs; From a76d83e6cdefeb0627f8e7654bb7d4aae5972931 Mon Sep 17 00:00:00 2001 From: Jacob Lee Date: Thu, 19 Dec 2024 17:58:43 -0800 Subject: [PATCH 11/13] fix(openai): Remove extra serialized params for OpenAI messages (#7414) --- libs/langchain-openai/src/chat_models.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/libs/langchain-openai/src/chat_models.ts b/libs/langchain-openai/src/chat_models.ts index 0906d2848655..3bd3a4e5ee57 100644 --- a/libs/langchain-openai/src/chat_models.ts +++ b/libs/langchain-openai/src/chat_models.ts @@ -15,6 +15,7 @@ import { OpenAIToolCall, isAIMessage, UsageMetadata, + BaseMessageFields, } from "@langchain/core/messages"; import { type ChatGeneration, @@ -1677,9 +1678,13 @@ export class ChatOpenAI< } // Fields are not serialized unless passed to the constructor // Doing this ensures all fields on the message are serialized - generation.message = new AIMessage({ - ...generation.message, - }); + generation.message = new AIMessage( + Object.fromEntries( + Object.entries(generation.message).filter( + ([key]) => !key.startsWith("lc_") + ) + ) as BaseMessageFields + ); generations.push(generation); } return { From a21aad9340c2a2f67063f3e82f3fdc368cbe8204 Mon Sep 17 00:00:00 2001 From: Jacob Lee Date: Thu, 19 Dec 2024 18:04:24 -0800 Subject: [PATCH 12/13] release(openai): 0.3.16 (#7416) --- libs/langchain-openai/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/langchain-openai/package.json b/libs/langchain-openai/package.json index 1d1b56cf7a7c..2d0766a6a4d0 100644 --- a/libs/langchain-openai/package.json +++ b/libs/langchain-openai/package.json @@ -1,6 +1,6 @@ { "name": "@langchain/openai", - "version": "0.3.15", + "version": "0.3.16", "description": "OpenAI integrations for LangChain.js", "type": "module", "engines": { From da80dce85760ff9a313be3b4aecc1ef9a264b913 Mon Sep 17 00:00:00 2001 From: Jacob Lee Date: Thu, 19 Dec 2024 18:12:11 -0800 Subject: [PATCH 13/13] fix(openai): Fix test (#7415) --- .../src/tests/chat_models-vision.int.test.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/libs/langchain-openai/src/tests/chat_models-vision.int.test.ts b/libs/langchain-openai/src/tests/chat_models-vision.int.test.ts index 23c00bbd0602..8486f438c8a9 100644 --- a/libs/langchain-openai/src/tests/chat_models-vision.int.test.ts +++ b/libs/langchain-openai/src/tests/chat_models-vision.int.test.ts @@ -10,7 +10,7 @@ test("Test ChatOpenAI with a file", async () => { const __dirname = path.dirname(__filename); const imageData = await fs.readFile(path.join(__dirname, "/data/hotdog.jpg")); const chat = new ChatOpenAI({ - modelName: "gpt-4-vision-preview", + modelName: "gpt-4o", maxTokens: 1024, }); const message = new HumanMessage({ @@ -35,19 +35,20 @@ test("Test ChatOpenAI with a file", async () => { test("Test ChatOpenAI with a URL", async () => { const chat = new ChatOpenAI({ - modelName: "gpt-4-vision-preview", + modelName: "gpt-4o", maxTokens: 1024, }); const message = new HumanMessage({ content: [ { type: "text", - text: "What does this image say?", + text: "What's in this image?", }, { type: "image_url", - image_url: - "https://www.freecodecamp.org/news/content/images/2023/05/Screenshot-2023-05-29-at-5.40.38-PM.png", + image_url: { + url: "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", + }, }, ], });