diff --git a/.github/workflows/standard-tests.yml b/.github/workflows/standard-tests.yml new file mode 100644 index 000000000000..d474706a7597 --- /dev/null +++ b/.github/workflows/standard-tests.yml @@ -0,0 +1,92 @@ +name: Standard Tests (Integration) + +on: + workflow_dispatch: + schedule: + - cron: '0 13 * * *' + +jobs: + standard-tests: + runs-on: ubuntu-latest + strategy: + matrix: + package: [anthropic, cohere, google-genai, groq, mistralai] + steps: + - uses: actions/checkout@v4 + - name: Use Node.js 18.x + uses: actions/setup-node@v3 + with: + node-version: 18.x + cache: "yarn" + - name: Install dependencies + run: yarn install --immutable --mode=skip-build + - name: Run standard tests (integration) for ${{ matrix.package }} + run: yarn test:standard:int --filter=@langchain/${{ matrix.package }} + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} + MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} + + # The `@langchain/openai` package contains standard tests for ChatOpenAI and AzureChatOpenAI + # We want to run these separately, so we need to pass the exact path for each test, which means + # we need separate jobs for each test. + standard-tests-openai: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Use Node.js 18.x + uses: actions/setup-node@v3 + with: + node-version: 18.x + cache: "yarn" + - name: Install dependencies + run: yarn install --immutable --mode=skip-build + - name: Build `@langchain/openai` + run: yarn build --filter=@langchain/openai + - name: Run standard tests (integration) for ChatOpenAI + run: yarn workspace @langchain/openai test:single src/tests/chat_models.standard.int.test.ts + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + + standard-tests-azure-openai: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Use Node.js 18.x + uses: actions/setup-node@v3 + with: + node-version: 18.x + cache: "yarn" + - name: Install dependencies + run: yarn install --immutable --mode=skip-build + - name: Build `@langchain/openai` + run: yarn build --filter=@langchain/openai + - name: Run standard tests (integration) for `@langchain/openai` AzureChatOpenAI + run: yarn workspace @langchain/openai test:single src/tests/azure/chat_models.standard.int.test.ts + env: + AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} + AZURE_OPENAI_API_DEPLOYMENT_NAME: "chat" + AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION }} + AZURE_OPENAI_BASE_PATH: ${{ secrets.AZURE_OPENAI_BASE_PATH }} + + standard-tests-bedrock: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Use Node.js 18.x + uses: actions/setup-node@v3 + with: + node-version: 18.x + cache: "yarn" + - name: Install dependencies + run: yarn install --immutable --mode=skip-build + - name: Build `@langchain/community` + run: yarn build --filter=@langchain/community + - name: Run standard tests (integration) for `@langchain/community` BedrockChat + run: yarn workspace @langchain/community test:single src/chat_models/tests/chatbedrock.standard.int.test.ts + env: + BEDROCK_AWS_REGION: "us-east-1" + BEDROCK_AWS_SECRET_ACCESS_KEY: ${{ secrets.BEDROCK_AWS_SECRET_ACCESS_KEY }} + BEDROCK_AWS_ACCESS_KEY_ID: ${{ secrets.BEDROCK_AWS_ACCESS_KEY_ID }} diff --git a/docs/core_docs/docs/concepts.mdx b/docs/core_docs/docs/concepts.mdx index fd9f55005442..fd02e5f694d5 100644 --- a/docs/core_docs/docs/concepts.mdx +++ b/docs/core_docs/docs/concepts.mdx @@ -1,3 +1,34 @@ +--- +keywords: + [ + prompt, + prompttemplate, + chatprompttemplate, + tool, + tools, + runnable, + runnables, + invoke, + vector, + vectorstore, + vectorstores, + embedding, + embeddings, + chat, + chat model, + llm, + llms, + retriever, + retrievers, + loader, + loaders, + document, + documents, + output, + output parser, + ] +--- + # Conceptual guide This section contains introductions to key parts of LangChain. @@ -641,8 +672,9 @@ LangChain provides several advanced retrieval types. A full list is below, along | [Multi Vector](/docs/how_to/multi_vector/) | Vectorstore + Document Store | Sometimes during indexing | If you are able to extract information from documents that you think is more relevant to index than the text itself. | This involves creating multiple vectors for each document. Each vector could be created in a myriad of ways - examples include summaries of the text and hypothetical questions. | | [Self Query](/docs/how_to/self_query/) | Vectorstore | Yes | If users are asking questions that are better answered by fetching documents based on metadata rather than similarity with the text. | This uses an LLM to transform user input into two things: (1) a string to look up semantically, (2) a metadata filer to go along with it. This is useful because oftentimes questions are about the METADATA of documents (not the content itself). | | [Contextual Compression](/docs/how_to/contextual_compression/) | Any | Sometimes | If you are finding that your retrieved documents contain too much irrelevant information and are distracting the LLM. | This puts a post-processing step on top of another retriever and extracts only the most relevant information from retrieved documents. This can be done with embeddings or an LLM. | -| [Time-Weighted Vectorstore](/docs/how_to/time_weighted_vectorstore/) | Vectorstore | No | If you have timestamps associated with your documents, and you want to retrieve the most recent ones | This fetches documents based on a combination of semantic similarity (as in normal vector retrieval) and recency (looking at timestamps of indexed documents) | -| [Multi-Query Retriever](/docs/how_to/multiple_queries/) | Any | Yes | If users are asking questions that are complex and require multiple pieces of distinct information to respond | This uses an LLM to generate multiple queries from the original one. This is useful when the original query needs pieces of information about multiple topics to be properly answered. By generating multiple queries, we can then fetch documents for each of them. | +| [Time-Weighted Vectorstore](/docs/how_to/time_weighted_vectorstore/) | Vectorstore | No | If you have timestamps associated with your documents, and you want to retrieve the most recent ones. | This fetches documents based on a combination of semantic similarity (as in normal vector retrieval) and recency (looking at timestamps of indexed documents) | +| [Multi-Query Retriever](/docs/how_to/multiple_queries/) | Any | Yes | If users are asking questions that are complex and require multiple pieces of distinct information to respond. | This uses an LLM to generate multiple queries from the original one. This is useful when the original query needs pieces of information about multiple topics to be properly answered. By generating multiple queries, we can then fetch documents for each of them. | +| [Ensemble](/docs/how_to/ensemble_retriever) | Any | No | If you have multiple retrieval methods and want to try combining them. | This fetches documents from multiple retrievers and then combines them. | ### Text splitting diff --git a/docs/core_docs/docs/how_to/document_loader_html.ipynb b/docs/core_docs/docs/how_to/document_loader_html.ipynb index e7116a606924..ee037f3b937e 100644 --- a/docs/core_docs/docs/how_to/document_loader_html.ipynb +++ b/docs/core_docs/docs/how_to/document_loader_html.ipynb @@ -45,7 +45,7 @@ "1. Download & start the Docker container:\n", " \n", "```bash\n", - "docker run -p 8000:8000 -d --rm --name unstructured-api quay.io/unstructured-io/unstructured-api:latest --port 8000 --host 0.0.0.0\n", + "docker run -p 8000:8000 -d --rm --name unstructured-api downloads.unstructured.io/unstructured-io/unstructured-api:latest --port 8000 --host 0.0.0.0\n", "```\n", "\n", "2. Get a free API key & API URL [here](https://unstructured.io/api-key), and set it in your environment (as per the Unstructured website, it may take up to an hour to allocate your API key & URL.):\n", diff --git a/docs/core_docs/docs/how_to/document_loader_markdown.ipynb b/docs/core_docs/docs/how_to/document_loader_markdown.ipynb index a72849c739df..d34914280fbc 100644 --- a/docs/core_docs/docs/how_to/document_loader_markdown.ipynb +++ b/docs/core_docs/docs/how_to/document_loader_markdown.ipynb @@ -50,7 +50,7 @@ "1. Download & start the Docker container:\n", " \n", "```bash\n", - "docker run -p 8000:8000 -d --rm --name unstructured-api quay.io/unstructured-io/unstructured-api:latest --port 8000 --host 0.0.0.0\n", + "docker run -p 8000:8000 -d --rm --name unstructured-api downloads.unstructured.io/unstructured-io/unstructured-api:latest --port 8000 --host 0.0.0.0\n", "```\n", "\n", "2. Get a free API key & API URL [here](https://unstructured.io/api-key), and set it in your environment (as per the Unstructured website, it may take up to an hour to allocate your API key & URL.):\n", diff --git a/docs/core_docs/docs/how_to/ensemble_retriever.mdx b/docs/core_docs/docs/how_to/ensemble_retriever.mdx new file mode 100644 index 000000000000..6eaf23871a33 --- /dev/null +++ b/docs/core_docs/docs/how_to/ensemble_retriever.mdx @@ -0,0 +1,29 @@ +# How to combine results from multiple retrievers + +:::info Prerequisites + +This guide assumes familiarity with the following concepts: + +- [Documents](/docs/concepts#document) +- [Retrievers](/docs/concepts#retrievers) + +::: + +The [EnsembleRetriever](https://api.js.langchain.com/classes/langchain_retrievers_ensemble.EnsembleRetriever.html) supports ensembling of results from multiple retrievers. It is initialized with a list of [BaseRetriever](https://api.js.langchain.com/classes/langchain_core_retrievers.BaseRetriever.html) objects. EnsembleRetrievers rerank the results of the constituent retrievers based on the [Reciprocal Rank Fusion](https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf) algorithm. + +By leveraging the strengths of different algorithms, the `EnsembleRetriever` can achieve better performance than any single algorithm. + +One useful pattern is to combine a keyword matching retriever with a dense retriever (like embedding similarity), because their strengths are complementary. This can be considered a form of "hybrid search". The sparse retriever is good at finding relevant documents based on keywords, while the dense retriever is good at finding relevant documents based on semantic similarity. + +Below we demonstrate ensembling of a [simple custom retriever](/docs/how_to/custom_retriever/) that simply returns documents that directly contain the input query with a retriever derived from a [demo, in-memory, vector store](https://api.js.langchain.com/classes/langchain_vectorstores_memory.MemoryVectorStore.html). + +import CodeBlock from "@theme/CodeBlock"; +import Example from "@examples/retrievers/ensemble_retriever.ts"; + +{Example} + +## Next steps + +You've now learned how to combine results from multiple retrievers. +Next, check out some other retrieval how-to guides, such as how to [improve results using multiple embeddings per document](/docs/how_to/multi_vector) +or how to [create your own custom retriever](/docs/how_to/custom_retriever). diff --git a/docs/core_docs/docs/how_to/index.mdx b/docs/core_docs/docs/how_to/index.mdx index 969e99689af7..4ac9ef722ae8 100644 --- a/docs/core_docs/docs/how_to/index.mdx +++ b/docs/core_docs/docs/how_to/index.mdx @@ -133,6 +133,7 @@ Retrievers are responsible for taking a query and returning relevant documents. - [How to: generate multiple queries to retrieve data for](/docs/how_to/multiple_queries) - [How to: use contextual compression to compress the data retrieved](/docs/how_to/contextual_compression) - [How to: write a custom retriever class](/docs/how_to/custom_retriever) +- [How to: combine the results from multiple retrievers](/docs/how_to/ensemble_retriever) - [How to: generate multiple embeddings per document](/docs/how_to/multi_vector) - [How to: retrieve the whole document for a chunk](/docs/how_to/parent_document_retriever) - [How to: generate metadata filters](/docs/how_to/self_query) diff --git a/docs/core_docs/docs/how_to/recursive_text_splitter.ipynb b/docs/core_docs/docs/how_to/recursive_text_splitter.ipynb index c6f1b0bbb96c..63741f503405 100644 --- a/docs/core_docs/docs/how_to/recursive_text_splitter.ipynb +++ b/docs/core_docs/docs/how_to/recursive_text_splitter.ipynb @@ -1,5 +1,18 @@ { "cells": [ + { + "cell_type": "raw", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "---\n", + "keywords: [recursivecharactertextsplitter]\n", + "---" + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/docs/core_docs/docs/how_to/sequence.ipynb b/docs/core_docs/docs/how_to/sequence.ipynb index 8b83841eb961..d5840412346f 100644 --- a/docs/core_docs/docs/how_to/sequence.ipynb +++ b/docs/core_docs/docs/how_to/sequence.ipynb @@ -2,10 +2,14 @@ "cells": [ { "cell_type": "raw", - "metadata": {}, + "metadata": { + "vscode": { + "languageId": "raw" + } + }, "source": [ "---\n", - "keywords: [Runnable, Runnables, LCEL]\n", + "keywords: [chain, chaining, runnablesequence]\n", "---" ] }, diff --git a/docs/core_docs/docs/how_to/tool_calling.ipynb b/docs/core_docs/docs/how_to/tool_calling.ipynb index cf854060d5ff..ebf40e2a9eff 100644 --- a/docs/core_docs/docs/how_to/tool_calling.ipynb +++ b/docs/core_docs/docs/how_to/tool_calling.ipynb @@ -1,5 +1,18 @@ { "cells": [ + { + "cell_type": "raw", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "---\n", + "keywords: [function, function calling, tool, tool calling]\n", + "---" + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/docs/core_docs/docs/integrations/chat/ollama_functions.mdx b/docs/core_docs/docs/integrations/chat/ollama_functions.mdx index eb66ba383283..fece11d8de1b 100644 --- a/docs/core_docs/docs/integrations/chat/ollama_functions.mdx +++ b/docs/core_docs/docs/integrations/chat/ollama_functions.mdx @@ -19,7 +19,7 @@ Follow [these instructions](https://github.com/jmorganca/ollama) to set up and r You can initialize this wrapper the same way you'd initialize a standard `ChatOllama` instance: ```typescript -import { OllamaFunctions } from "langchain/experimental/chat_models/ollama_functions"; +import { OllamaFunctions } from "@langchain/community/experimental/chat_models/ollama_functions"; const model = new OllamaFunctions({ temperature: 0.1, diff --git a/docs/core_docs/docs/integrations/document_loaders/file_loaders/unstructured.mdx b/docs/core_docs/docs/integrations/document_loaders/file_loaders/unstructured.mdx index 1db153973a33..1626b3987dff 100644 --- a/docs/core_docs/docs/integrations/document_loaders/file_loaders/unstructured.mdx +++ b/docs/core_docs/docs/integrations/document_loaders/file_loaders/unstructured.mdx @@ -11,7 +11,7 @@ This example covers how to use Unstructured to load files of many types. Unstruc You can run Unstructured locally in your computer using Docker. To do so, you need to have Docker installed. You can find the instructions to install Docker [here](https://docs.docker.com/get-docker/). ```bash -docker run -p 8000:8000 -d --rm --name unstructured-api quay.io/unstructured-io/unstructured-api:latest --port 8000 --host 0.0.0.0 +docker run -p 8000:8000 -d --rm --name unstructured-api downloads.unstructured.io/unstructured-io/unstructured-api:latest --port 8000 --host 0.0.0.0 ``` ## Usage diff --git a/docs/core_docs/docs/integrations/platforms/aws.mdx b/docs/core_docs/docs/integrations/platforms/aws.mdx index 2ca7f358fbd0..f8d06c34ac56 100644 --- a/docs/core_docs/docs/integrations/platforms/aws.mdx +++ b/docs/core_docs/docs/integrations/platforms/aws.mdx @@ -1,3 +1,7 @@ +--- +keywords: [bedrock] +--- + # AWS All functionality related to [Amazon AWS](https://aws.amazon.com/) platform diff --git a/docs/core_docs/docs/integrations/platforms/microsoft.mdx b/docs/core_docs/docs/integrations/platforms/microsoft.mdx index 6518a36baab1..02e32eaa4eee 100644 --- a/docs/core_docs/docs/integrations/platforms/microsoft.mdx +++ b/docs/core_docs/docs/integrations/platforms/microsoft.mdx @@ -1,3 +1,7 @@ +--- +keywords: [azure] +--- + import CodeBlock from "@theme/CodeBlock"; # Microsoft diff --git a/docs/core_docs/docs/integrations/platforms/openai.mdx b/docs/core_docs/docs/integrations/platforms/openai.mdx index 6f1d32437fb8..127ad4929c9f 100644 --- a/docs/core_docs/docs/integrations/platforms/openai.mdx +++ b/docs/core_docs/docs/integrations/platforms/openai.mdx @@ -1,3 +1,7 @@ +--- +keywords: [openai] +--- + # OpenAI All functionality related to OpenAI diff --git a/docs/core_docs/docs/integrations/text_embedding/openai.mdx b/docs/core_docs/docs/integrations/text_embedding/openai.mdx index 5e81a9219cc8..bba94b0777ee 100644 --- a/docs/core_docs/docs/integrations/text_embedding/openai.mdx +++ b/docs/core_docs/docs/integrations/text_embedding/openai.mdx @@ -1,3 +1,7 @@ +--- +keywords: [openaiembeddings] +--- + # OpenAI The `OpenAIEmbeddings` class uses the OpenAI API to generate embeddings for a given text. By default it strips new line characters from the text, as recommended by OpenAI, but you can disable this by passing `stripNewLines: false` to the constructor. diff --git a/docs/core_docs/docs/integrations/vectorstores/googlevertexai.mdx b/docs/core_docs/docs/integrations/vectorstores/googlevertexai.mdx index e7509a6cd08e..7636ef2afeb3 100644 --- a/docs/core_docs/docs/integrations/vectorstores/googlevertexai.mdx +++ b/docs/core_docs/docs/integrations/vectorstores/googlevertexai.mdx @@ -63,10 +63,10 @@ initial testing, you will want to use something like a [GoogleCloudStorageDocstore](https://v02.api.js.langchain.com/classes/langchain_stores_doc_gcs.GoogleCloudStorageDocstore.html) to store it more permanently. ```typescript -import { MatchingEngine } from "langchain/vectorstores/googlevertexai"; +import { MatchingEngine } from "@langchain/community/vectorstores/googlevertexai"; import { Document } from "langchain/document"; import { SyntheticEmbeddings } from "langchain/embeddings/fake"; -import { GoogleCloudStorageDocstore } from "langchain/stores/doc/gcs"; +import { GoogleCloudStorageDocstore } from "@langchain/community/stores/doc/gcs"; const embeddings = new SyntheticEmbeddings({ vectorSize: Number.parseInt( diff --git a/docs/core_docs/docs/langgraph.mdx b/docs/core_docs/docs/langgraph.mdx deleted file mode 100644 index 345a228f5526..000000000000 --- a/docs/core_docs/docs/langgraph.mdx +++ /dev/null @@ -1,848 +0,0 @@ -# 🦜🕸️LangGraph.js - -⚡ Building language agents as graphs ⚡ - -## Overview - -LangGraph is a library for building stateful, multi-actor applications with LLMs, built on top of (and intended to be used with) [LangChain.js](https://github.com/langchain-ai/langchainjs). -It extends the [LangChain Expression Language](/docs/how_to/#langchain-expression-language-lcel) with the ability to coordinate multiple chains (or actors) across multiple steps of computation in a cyclic manner. -It is inspired by [Pregel](https://research.google/pubs/pub37252/) and [Apache Beam](https://beam.apache.org/). -The current interface exposed is one inspired by [NetworkX](https://networkx.org/documentation/latest/). - -The main use is for adding **cycles** to your LLM application. -Crucially, LangGraph is NOT optimized for only **DAG** workflows. -If you want to build a DAG, you should use just use [LangChain Expression Language](/docs/how_to/#langchain-expression-language-lcel). - -Cycles are important for agent-like behaviors, where you call an LLM in a loop, asking it what action to take next. - -> Looking for the Python version? Click [here](https://github.com/langchain-ai/langgraph). - -## Installation - -```bash -npm install @langchain/langgraph -``` - -## Quick start - -One of the central concepts of LangGraph is state. Each graph execution creates a state that is passed between nodes in the graph as they execute, and each node updates this internal state with its return value after it executes. The way that the graph updates its internal state is defined by either the type of graph chosen or a custom function. - -State in LangGraph can be pretty general, but to keep things simpler to start, we'll show off an example where the graph's state is limited to a list of chat messages using the built-in `MessageGraph` class. This is convenient when using LangGraph with LangChain chat models because we can return chat model output directly. - -First, install the LangChain OpenAI integration package: - -```shell -npm i @langchain/openai -``` - -We also need to export some environment variables: - -```shell -export OPENAI_API_KEY=sk-... -``` - -And now we're ready! The graph below contains a single node called `"oracle"` that executes a chat model, then returns the result: - -```ts -import { ChatOpenAI } from "@langchain/openai"; -import { HumanMessage, BaseMessage } from "@langchain/core/messages"; -import { END, MessageGraph } from "@langchain/langgraph"; - -const model = new ChatOpenAI({ temperature: 0 }); - -const graph = new MessageGraph(); - -graph.addNode("oracle", async (state: BaseMessage[]) => { - return model.invoke(state); -}); - -graph.addEdge("oracle", END); - -graph.setEntryPoint("oracle"); - -const runnable = graph.compile(); -``` - -Let's run it! - -```ts -// For Message graph, input should always be a message or list of messages. -const res = await runnable.invoke(new HumanMessage("What is 1 + 1?")); -``` - -```ts -[ - HumanMessage { - content: 'What is 1 + 1?', - additional_kwargs: {} - }, - AIMessage { - content: '1 + 1 equals 2.', - additional_kwargs: { function_call: undefined, tool_calls: undefined } - } -] -``` - -So what did we do here? Let's break it down step by step: - -1. First, we initialize our model and a `MessageGraph`. -2. Next, we add a single node to the graph, called `"oracle"`, which simply calls the model with the given input. -3. We add an edge from this `"oracle"` node to the special value `END`. This means that execution will end after current node. -4. We set `"oracle"` as the entrypoint to the graph. -5. We compile the graph, ensuring that no more modifications to it can be made. - -Then, when we execute the graph: - -1. LangGraph adds the input message to the internal state, then passes the state to the entrypoint node, `"oracle"`. -2. The `"oracle"` node executes, invoking the chat model. -3. The chat model returns an `AIMessage`. LangGraph adds this to the state. -4. Execution progresses to the special `END` value and outputs the final state. - -And as a result, we get a list of two chat messages as output. - -### Interaction with LCEL - -As an aside for those already familiar with LangChain - `addNode` actually takes any runnable as input. In the above example, the passed function is automatically converted, but we could also have passed the model directly: - -```ts -graph.addNode("oracle", model); -``` - -In which case the `.invoke()` method will be called when the graph executes. - -Just make sure you are mindful of the fact that the input to the runnable is the entire current state. So this will fail: - -```ts -// This will NOT work with MessageGraph! -import { - ChatPromptTemplate, - MessagesPlaceholder, -} from "@langchain/core/prompts"; - -const prompt = ChatPromptTemplate.fromMessages([ - ["system", "You are a helpful assistant who always speaks in pirate dialect"], - MessagesPlaceholder("messages"), -]); - -const chain = prompt.pipe(model); - -// State is a list of messages, but our chain expects an object input: -// -// { messages: [] } -// -// Therefore, the graph will throw an exception when it executes here. -graph.addNode("oracle", chain); -``` - -## Conditional edges - -Now, let's move onto something a little bit less trivial. Because math can be difficult for LLMs, let's allow the LLM to conditionally call a calculator node using tool calling. - -```bash -npm i langchain @langchain/openai -``` - -We'll recreate our graph with an additional `"calculator"` that will take the result of the most recent message, if it is a math expression, and calculate the result. -We'll also bind the calculator to the OpenAI model as a tool to allow the model to optionally use the tool if it deems necessary: - -```ts -import { ToolMessage } from "@langchain/core/messages"; -import { Calculator } from "langchain/tools/calculator"; -import { convertToOpenAITool } from "@langchain/core/utils/function_calling"; - -const model = new ChatOpenAI({ - temperature: 0, -}).bind({ - tools: [convertToOpenAITool(new Calculator())], - tool_choice: "auto", -}); - -const graph = new MessageGraph(); - -graph.addNode("oracle", async (state: BaseMessage[]) => { - return model.invoke(state); -}); - -graph.addNode("calculator", async (state: BaseMessage[]) => { - const tool = new Calculator(); - const toolCalls = state[state.length - 1].additional_kwargs.tool_calls ?? []; - const calculatorCall = toolCalls.find( - (toolCall) => toolCall.function.name === "calculator" - ); - if (calculatorCall === undefined) { - throw new Error("No calculator input found."); - } - const result = await tool.invoke( - JSON.parse(calculatorCall.function.arguments) - ); - return new ToolMessage({ - tool_call_id: calculatorCall.id, - content: result, - }); -}); - -graph.addEdge("calculator", END); - -graph.setEntryPoint("oracle"); -``` - -Now let's think - what do we want to have happen? - -- If the `"oracle"` node returns a message expecting a tool call, we want to execute the `"calculator"` node -- If not, we can just end execution - -We can achieve this using **conditional edges**, which routes execution to a node based on the current state using a function. - -Here's what that looks like: - -```ts -const router = (state: BaseMessage[]) => { - const toolCalls = state[state.length - 1].additional_kwargs.tool_calls ?? []; - if (toolCalls.length) { - return "calculator"; - } else { - return "end"; - } -}; - -graph.addConditionalEdges("oracle", router, { - calculator: "calculator", - end: END, -}); -``` - -If the model output contains a tool call, we move to the `"calculator"` node. Otherwise, we end. - -Great! Now all that's left is to compile the graph and try it out. Math-related questions are routed to the calculator tool: - -```ts -const runnable = graph.compile(); -const mathResponse = await runnable.invoke(new HumanMessage("What is 1 + 1?")); -``` - -```ts -[ - HumanMessage { - content: 'What is 1 + 1?', - additional_kwargs: {} - }, - AIMessage { - content: '', - additional_kwargs: { function_call: undefined, tool_calls: [Array] } - }, - ToolMessage { - content: '2', - name: undefined, - additional_kwargs: {}, - tool_call_id: 'call_P7KWQoftVsj6fgsqKyolWp91' - } -] -``` - -While conversational responses are outputted directly: - -```ts -const otherResponse = await runnable.invoke( - new HumanMessage("What is your name?") -); -``` - -```ts -[ - HumanMessage { - content: 'What is your name?', - additional_kwargs: {} - }, - AIMessage { - content: 'My name is Assistant. How can I assist you today?', - additional_kwargs: { function_call: undefined, tool_calls: undefined } - } -] -``` - -## Cycles - -Now, let's go over a more general example with a cycle. We will recreate the [`AgentExecutor`](/docs/how_to/agent_executor/) class from LangChain. - -The benefits of creating it with LangGraph is that it is more modifiable. - -We will need to install some LangChain packages: - -```shell -npm install langchain @langchain/core @langchain/community @langchain/openai -``` - -We also need additional environment variables. - -```shell -export OPENAI_API_KEY=sk-... -export TAVILY_API_KEY=tvly-... -``` - -Optionally, we can set up [LangSmith](https://docs.smith.langchain.com/) for best-in-class observability. - -```shell -export LANGCHAIN_TRACING_V2="true" -export LANGCHAIN_API_KEY=ls__... -export LANGCHAIN_ENDPOINT=https://api.langchain.com -``` - -### Set up the tools - -As above, we will first define the tools we want to use. -For this simple example, we will use a built-in search tool via Tavily. -However, it is really easy to create your own tools - see documentation [here](/docs/how_to/custom_tools) on how to do that. - -```typescript -import { TavilySearchResults } from "@langchain/community/tools/tavily_search"; - -const tools = [new TavilySearchResults({ maxResults: 1 })]; -``` - -We can now wrap these tools in a ToolExecutor, which simply takes in a ToolInvocation and calls that tool, returning the output. - -A ToolInvocation is any type with `tool` and `toolInput` attribute. - -```typescript -import { ToolExecutor } from "@langchain/langgraph/prebuilt"; - -const toolExecutor = new ToolExecutor({ tools }); -``` - -### Set up the model - -Now we need to load the chat model we want to use. -This time, we'll use the older function calling interface. This walkthrough will use OpenAI, but we can choose any model that supports OpenAI function calling. - -```typescript -import { ChatOpenAI } from "@langchain/openai"; - -// We will set streaming: true so that we can stream tokens -// See the streaming section for more information on this. -const model = new ChatOpenAI({ - temperature: 0, - streaming: true, -}); -``` - -After we've done this, we should make sure the model knows that it has these tools available to call. -We can do this by converting the LangChain tools into the format for OpenAI function calling, and then bind them to the model class. - -```typescript -import { convertToOpenAIFunction } from "@langchain/core/utils/function_calling"; - -const toolsAsOpenAIFunctions = tools.map((tool) => - convertToOpenAIFunction(tool) -); -const newModel = model.bind({ - functions: toolsAsOpenAIFunctions, -}); -``` - -### Define the agent state - -This time, we'll use the more general `StateGraph`. -This graph is parameterized by a state object that it passes around to each node. -Remember that each node then returns operations to update that state. -These operations can either SET specific attributes on the state (e.g. overwrite the existing values) or ADD to the existing attribute. -Whether to set or add is denoted by annotating the state object you construct the graph with. - -For this example, the state we will track will just be a list of messages. -We want each node to just add messages to that list. -Therefore, we will use an object with one key (`messages`) with the value as an object: `{ value: Function, default?: () => any }` - -The `default` key must be a factory that returns the default value for that attribute. - -```typescript -import { BaseMessage } from "@langchain/core/messages"; - -const agentState = { - messages: { - value: (x: BaseMessage[], y: BaseMessage[]) => x.concat(y), - default: () => [], - }, -}; -``` - -You can think of the `MessageGraph` used in the initial example as a preconfigured version of this graph. The difference is that the state is directly a list of messages, -instead of an object containing a key called `"messages"` whose value is a list of messages. -The `MessageGraph` update step is similar to the one above where we always append the returned values of a node to the internal state. - -### Define the nodes - -We now need to define a few different nodes in our graph. -In LangGraph, a node can be either a function or a [runnable](/docs/how_to/#langchain-expression-language-lcel). -There are two main nodes we need for this: - -1. The agent: responsible for deciding what (if any) actions to take. -2. A function to invoke tools: if the agent decides to take an action, this node will then execute that action. - -We will also need to define some edges. -Some of these edges may be conditional. -The reason they are conditional is that based on the output of a node, one of several paths may be taken. -The path that is taken is not known until that node is run (the LLM decides). - -1. Conditional Edge: after the agent is called, we should either: - a. If the agent said to take an action, then the function to invoke tools should be called - b. If the agent said that it was finished, then it should finish -2. Normal Edge: after the tools are invoked, it should always go back to the agent to decide what to do next - -Let's define the nodes, as well as a function to decide how what conditional edge to take. - -```typescript -import { FunctionMessage } from "@langchain/core/messages"; -import { AgentAction } from "@langchain/core/agents"; -import { - ChatPromptTemplate, - MessagesPlaceholder, -} from "@langchain/core/prompts"; - -// Define the function that determines whether to continue or not -const shouldContinue = (state: { messages: Array }) => { - const { messages } = state; - const lastMessage = messages[messages.length - 1]; - // If there is no function call, then we finish - if ( - !("function_call" in lastMessage.additional_kwargs) || - !lastMessage.additional_kwargs.function_call - ) { - return "end"; - } - // Otherwise if there is, we continue - return "continue"; -}; - -// Define the function to execute tools -const _getAction = (state: { messages: Array }): AgentAction => { - const { messages } = state; - // Based on the continue condition - // we know the last message involves a function call - const lastMessage = messages[messages.length - 1]; - if (!lastMessage) { - throw new Error("No messages found."); - } - if (!lastMessage.additional_kwargs.function_call) { - throw new Error("No function call found in message."); - } - // We construct an AgentAction from the function_call - return { - tool: lastMessage.additional_kwargs.function_call.name, - toolInput: JSON.parse( - lastMessage.additional_kwargs.function_call.arguments - ), - log: "", - }; -}; - -// Define the function that calls the model -const callModel = async (state: { messages: Array }) => { - const { messages } = state; - // You can use a prompt here to tweak model behavior. - // You can also just pass messages to the model directly. - const prompt = ChatPromptTemplate.fromMessages([ - ["system", "You are a helpful assistant."], - new MessagesPlaceholder("messages"), - ]); - const response = await prompt.pipe(newModel).invoke({ messages }); - // We return a list, because this will get added to the existing list - return { - messages: [response], - }; -}; - -const callTool = async (state: { messages: Array }) => { - const action = _getAction(state); - // We call the tool_executor and get back a response - const response = await toolExecutor.invoke(action); - // We use the response to create a FunctionMessage - const functionMessage = new FunctionMessage({ - content: response, - name: action.tool, - }); - // We return a list, because this will get added to the existing list - return { messages: [functionMessage] }; -}; -``` - -### Define the graph - -We can now put it all together and define the graph! - -```typescript -import { StateGraph, END } from "@langchain/langgraph"; - -// Define a new graph -const workflow = new StateGraph({ - channels: agentState, -}); - -// Define the two nodes we will cycle between -workflow.addNode("agent", callModel); -workflow.addNode("action", callTool); - -// Set the entrypoint as `agent` -// This means that this node is the first one called -workflow.setEntryPoint("agent"); - -// We now add a conditional edge -workflow.addConditionalEdges( - // First, we define the start node. We use `agent`. - // This means these are the edges taken after the `agent` node is called. - "agent", - // Next, we pass in the function that will determine which node is called next. - shouldContinue, - // Finally we pass in a mapping. - // The keys are strings, and the values are other nodes. - // END is a special node marking that the graph should finish. - // What will happen is we will call `should_continue`, and then the output of that - // will be matched against the keys in this mapping. - // Based on which one it matches, that node will then be called. - { - // If `tools`, then we call the tool node. - continue: "action", - // Otherwise we finish. - end: END, - } -); - -// We now add a normal edge from `tools` to `agent`. -// This means that after `tools` is called, `agent` node is called next. -workflow.addEdge("action", "agent"); - -// Finally, we compile it! -// This compiles it into a LangChain Runnable, -// meaning you can use it as you would any other runnable -const app = workflow.compile(); -``` - -### Use it! - -We can now use it! -This now exposes the [same interface](/docs/how_to/#langchain-expression-language-lcel) as all other LangChain runnables. -This runnable accepts a list of messages. - -```typescript -import { HumanMessage } from "@langchain/core/messages"; - -const inputs = { - messages: [new HumanMessage("what is the weather in sf")], -}; -const result = await app.invoke(inputs); -``` - -See a LangSmith trace of this run [here](https://smith.langchain.com/public/144af8a3-b496-43aa-ba9d-f0d5894196e2/r). - -This may take a little bit - it's making a few calls behind the scenes. -In order to start seeing some intermediate results as they happen, we can use streaming - see below for more information on that. - -## Streaming - -LangGraph has support for several different types of streaming. - -### Streaming Node Output - -One of the benefits of using LangGraph is that it is easy to stream output as it's produced by each node. - -```typescript -const inputs = { - messages: [new HumanMessage("what is the weather in sf")], -}; -for await (const output of await app.stream(inputs)) { - console.log("output", output); - console.log("-----\n"); -} -``` - -See a LangSmith trace of this run [here](https://smith.langchain.com/public/968cd1bf-0db2-410f-a5b4-0e73066cf06e/r). - -## Running Examples - -You can find some more example notebooks of different use-cases in the `examples/` folder in this repo. These example notebooks use the [Deno runtime](https://deno.land/). - -To pull in environment variables, you can create a `.env` file at the **root** of this repo (not in the `examples/` folder itself). - -## When to Use - -When should you use this versus [LangChain Expression Language](/docs/how_to/#langchain-expression-language-lcel)? - -If you need cycles. - -Langchain Expression Language allows you to easily define chains (DAGs) but does not have a good mechanism for adding in cycles. -`langgraph` adds that syntax. - -## Examples - -### ChatAgentExecutor: with function calling - -This agent executor takes a list of messages as input and outputs a list of messages. -All agent state is represented as a list of messages. -This specifically uses OpenAI function calling. -This is recommended agent executor for newer chat based models that support function calling. - -- [Getting Started Notebook](https://github.com/langchain-ai/langgraphjs/tree/main/examples/chat_agent_executor_with_function_calling/base.ipynb): Walks through creating this type of executor from scratch - -### AgentExecutor - -This agent executor uses existing LangChain agents. - -- [Getting Started Notebook](https://github.com/langchain-ai/langgraphjs/tree/main/examples/agent_executor/base.ipynb): Walks through creating this type of executor from scratch - -### Multi-agent Examples - -- [Multi-agent collaboration](https://github.com/langchain-ai/langgraphjs/tree/main/examples/multi_agent/multi_agent_collaboration.ipynb): how to create two agents that work together to accomplish a task -- [Multi-agent with supervisor](https://github.com/langchain-ai/langgraphjs/tree/main/examples/multi_agent/agent_supervisor.ipynb): how to orchestrate individual agents by using an LLM as a "supervisor" to distribute work -- [Hierarchical agent teams](https://github.com/langchain-ai/langgraphjs/tree/main/examples/multi_agent/hierarchical_agent_teams.ipynb): how to orchestrate "teams" of agents as nested graphs that can collaborate to solve a problem - -## Documentation - -There are only a few new APIs to use. - -### StateGraph - -The main entrypoint is `StateGraph`. - -```typescript -import { StateGraph } from "@langchain/langgraph"; -``` - -This class is responsible for constructing the graph. -It exposes an interface inspired by [NetworkX](https://networkx.org/documentation/latest/). -This graph is parameterized by a state object that it passes around to each node. - -#### `constructor` - -```typescript -interface StateGraphArgs { - channels: Record< - string, - { - value: BinaryOperator | null; - default?: () => T; - } - >; -} - -class StateGraph extends Graph { - constructor(fields: StateGraphArgs) {} -``` - -When constructing the graph, you need to pass in a schema for a state. -Each node then returns operations to update that state. -These operations can either SET specific attributes on the state (e.g. overwrite the existing values) or ADD to the existing attribute. -Whether to set or add is denoted by annotating the state object you construct the graph with. - -Let's take a look at an example: - -```typescript -import { BaseMessage } from "@langchain/core/messages"; - -const schema = { - input: { - value: null, - }, - agentOutcome: { - value: null, - }, - steps: { - value: (x: Array, y: Array) => x.concat(y), - default: () => [], - }, -}; -``` - -We can then use this like: - -```typescript -// Initialize the StateGraph with this state -const graph = new StateGraph({ channels: schema }) -// Create nodes and edges -... -// Compile the graph -const app = graph.compile() - -// The inputs should be an object, because the schema is an object -const inputs = { - // Let's assume this the input - input: "hi" - // Let's assume agent_outcome is set by the graph as some point - // It doesn't need to be provided, and it will be null by default -} -``` - -### `.addNode` - -```typescript -addNode(key: string, action: RunnableLike): void -``` - -This method adds a node to the graph. -It takes two arguments: - -- `key`: A string representing the name of the node. This must be unique. -- `action`: The action to take when this node is called. This should either be a function or a runnable. - -### `.addEdge` - -```typescript -addEdge(startKey: string, endKey: string): void -``` - -Creates an edge from one node to the next. -This means that output of the first node will be passed to the next node. -It takes two arguments. - -- `startKey`: A string representing the name of the start node. This key must have already been registered in the graph. -- `endKey`: A string representing the name of the end node. This key must have already been registered in the graph. - -### `.addConditionalEdges` - -```typescript -addConditionalEdges( - startKey: string, - condition: CallableFunction, - conditionalEdgeMapping: Record -): void -``` - -This method adds conditional edges. -What this means is that only one of the downstream edges will be taken, and which one that is depends on the results of the start node. -This takes three arguments: - -- `startKey`: A string representing the name of the start node. This key must have already been registered in the graph. -- `condition`: A function to call to decide what to do next. The input will be the output of the start node. It should return a string that is present in `conditionalEdgeMapping` and represents the edge to take. -- `conditionalEdgeMapping`: A mapping of string to string. The keys should be strings that may be returned by `condition`. The values should be the downstream node to call if that condition is returned. - -### `.setEntryPoint` - -```typescript -setEntryPoint(key: string): void -``` - -The entrypoint to the graph. -This is the node that is first called. -It only takes one argument: - -- `key`: The name of the node that should be called first. - -### `.setFinishPoint` - -```typescript -setFinishPoint(key: string): void -``` - -This is the exit point of the graph. -When this node is called, the results will be the final result from the graph. -It only has one argument: - -- `key`: The name of the node that, when called, will return the results of calling it as the final output - -Note: This does not need to be called if at any point you previously created an edge (conditional or normal) to `END` - -### `END` - -```typescript -import { END } from "@langchain/langgraph"; -``` - -This is a special node representing the end of the graph. -This means that anything passed to this node will be the final output of the graph. -It can be used in two places: - -- As the `endKey` in `addEdge` -- As a value in `conditionalEdgeMapping` as passed to `addConditionalEdges` - -## When to Use - -When should you use this versus [LangChain Expression Language](/docs/how_to/#langchain-expression-language-lcel)? - -If you need cycles. - -Langchain Expression Language allows you to easily define chains (DAGs) but does not have a good mechanism for adding in cycles. -`langgraph` adds that syntax. - -## Examples - -### AgentExecutor - -See the above Quick Start for an example of re-creating the LangChain [`AgentExecutor`](/docs/how_to/agent_executor/) class. - -### Forced Function Calling - -One simple modification of the above Graph is to modify it such that a certain tool is always called first. -This can be useful if you want to enforce a certain tool is called, but still want to enable agentic behavior after the fact. - -Assuming you have done the above Quick Start, you can build off it like: - -#### Define the first tool call - -Here, we manually define the first tool call that we will make. -Notice that it does that same thing as `agent` would have done (adds the `agentOutcome` key). -This is so that we can easily plug it in. - -```typescript -import { AgentStep, AgentAction, AgentFinish } from "@langchain/core/agents"; - -// Define the data type that the agent will return. -type AgentData = { - input: string; - steps: Array; - agentOutcome?: AgentAction | AgentFinish; -}; - -const firstAgent = (inputs: AgentData) => { - const newInputs = inputs; - const action = { - // We force call this tool - tool: "tavily_search_results_json", - // We just pass in the `input` key to this tool - toolInput: newInputs.input, - log: "", - }; - newInputs.agentOutcome = action; - return newInputs; -}; -``` - -#### Create the graph - -We can now create a new graph with this new node - -```typescript -const workflow = new Graph(); - -// Add the same nodes as before, plus this "first agent" -workflow.addNode("firstAgent", firstAgent); -workflow.addNode("agent", agent); -workflow.addNode("tools", executeTools); - -// We now set the entry point to be this first agent -workflow.setEntryPoint("firstAgent"); - -// We define the same edges as before -workflow.addConditionalEdges("agent", shouldContinue, { - continue: "tools", - exit: END, -}); -workflow.addEdge("tools", "agent"); - -// We also define a new edge, from the "first agent" to the tools node -// This is so that we can call the tool -workflow.addEdge("firstAgent", "tools"); - -// We now compile the graph as before -const chain = workflow.compile(); -``` - -#### Use it! - -We can now use it as before! -Depending on whether or not the first tool call is actually useful, this may save you an LLM call or two. - -```typescript -const result = await chain.invoke({ - input: "what is the weather in sf", - steps: [], -}); -``` - -You can see a LangSmith trace of this chain [here](https://smith.langchain.com/public/2e0a089f-8c05-405a-8404-b0a60b79a84a/r). diff --git a/docs/core_docs/docs/langsmith.mdx b/docs/core_docs/docs/langsmith.mdx deleted file mode 100644 index d244670a1aad..000000000000 --- a/docs/core_docs/docs/langsmith.mdx +++ /dev/null @@ -1,15 +0,0 @@ -# 🦜🛠️ LangSmith - -[LangSmith](https://smith.langchain.com) helps you trace and evaluate your language model applications and intelligent agents to help you -move from prototype to production. - -To get started, please refer to the [LangSmith documentation](https://docs.smith.langchain.com/). - -For tutorials and other end-to-end examples demonstrating ways to integrate LangSmith in your workflow, -check out the [LangSmith Cookbook](https://github.com/langchain-ai/langsmith-cookbook). Some of the guides therein include: - -- Leveraging user feedback in your JS application ([link](https://github.com/langchain-ai/langsmith-cookbook/blob/main/feedback-examples/nextjs/README.md)). -- Building an automated feedback pipeline ([link](https://github.com/langchain-ai/langsmith-cookbook/blob/main/feedback-examples/algorithmic-feedback/algorithmic_feedback.ipynb)). -- How to evaluate and audit your RAG workflows ([link](https://github.com/langchain-ai/langsmith-cookbook/tree/main/testing-examples/qa-correctness)). -- How to fine-tune an LLM on real usage data ([link](https://github.com/langchain-ai/langsmith-cookbook/blob/main/fine-tuning-examples/export-to-openai/fine-tuning-on-chat-runs.ipynb)). -- How to use the [LangChain Hub](https://smith.langchain.com/hub) to version your prompts ([link](https://github.com/langchain-ai/langsmith-cookbook/blob/main/hub-examples/retrieval-qa-chain/retrieval-qa.ipynb)) diff --git a/docs/core_docs/docs/tutorials/index.mdx b/docs/core_docs/docs/tutorials/index.mdx index b7618ee0429c..3eaef3554be2 100644 --- a/docs/core_docs/docs/tutorials/index.mdx +++ b/docs/core_docs/docs/tutorials/index.mdx @@ -9,7 +9,7 @@ New to LangChain or to LLM app development in general? Read this material to qui ### Basics -- [Build a Simple LLM Application](/docs/tutorials/llm_chain) +- [Build a Simple LLM Application with LCEL](/docs/tutorials/llm_chain) - [Build a Chatbot](/docs/tutorials/chatbot) - [Build an Agent](/docs/tutorials/agents) diff --git a/docs/core_docs/docs/tutorials/llm_chain.ipynb b/docs/core_docs/docs/tutorials/llm_chain.ipynb index fe7534afd0ce..d5a83e50d681 100644 --- a/docs/core_docs/docs/tutorials/llm_chain.ipynb +++ b/docs/core_docs/docs/tutorials/llm_chain.ipynb @@ -15,23 +15,21 @@ "id": "9316da0d", "metadata": {}, "source": [ - "# Build a Simple LLM Application\n", + "# Build a Simple LLM Application with LCEL\n", "\n", - "In this quickstart we'll show you how to build a simple LLM application. This application will translate text from English into another language. This is a relatively simple LLM application - it's just a single LLM call plus some prompting. Still, this is a great way to get started with LangChain - a lot of features can be built with just some prompting and an LLM call!\n", + "In this quickstart we'll show you how to build a simple LLM application with LangChain. This application will translate text from English into another language. This is a relatively simple LLM application - it's just a single LLM call plus some prompting. Still, this is a great way to get started with LangChain - a lot of features can be built with just some prompting and an LLM call!\n", "\n", - "## Concepts\n", - "\n", - "Concepts we will cover are:\n", + "After reading this tutorial, you'll have a high level overview of:\n", "\n", "- Using [language models](/docs/concepts/#chat-models)\n", "\n", "- Using [PromptTemplates](/docs/concepts/#prompt-templates) and [OutputParsers](/docs/concepts/#output-parsers)\n", "\n", - "- [Chaining](/docs/concepts/#langchain-expression-language) a PromptTemplate + LLM + OutputParser using LangChain\n", + "- Using [LangChain Expression Language (LCEL)](/docs/concepts/#langchain-expression-language) to chain components together\n", "\n", "- Debugging and tracing your application using [LangSmith](/docs/concepts/#langsmith)\n", "\n", - "That's a fair amount to cover! Let's dive in.\n", + "Let's dive in!\n", "\n", "## Setup\n", "\n", @@ -71,11 +69,6 @@ "id": "e5558ca9", "metadata": {}, "source": [ - "## Detailed walkthrough\n", - "\n", - "In this guide we will build an application to translate user input from one language to another.\n", - "\n", - "\n", "## Using Language Models\n", "\n", "First up, let's learn how to use a language model by itself. LangChain supports many different language models that you can use interchangably - select the one you want to use below!\n", @@ -218,9 +211,11 @@ "id": "d508b79d", "metadata": {}, "source": [ - "More commonly, we can \"chain\" the model with this output parser. This means this output parser will get called every time in this chain. This chain takes on the input type of the language model (string or list of message) and returns the output type of the output parser (string).\n", + "## Chaining together components with LCEL\n", "\n", - "We can easily create the chain using the `.pipe` method. The `.pipe` method is used in LangChain to combine two elements together." + "We can also \"chain\" the model to the output parser. This means this output parser will get called with the output from the model. This chain takes on the input type of the language model (string or list of message) and returns the output type of the output parser (string).\n", + "\n", + "We can create the chain using the `.pipe()` method. The `.pipe()` method is used in LangChain to combine two elements together." ] }, { @@ -259,6 +254,8 @@ "id": "dd009096", "metadata": {}, "source": [ + "This is a simple example of using [LangChain Expression Language (LCEL)](/docs/concepts/#langchain-expression-language) to chain together LangChain modules. There are several benefits to this approach, including optimized streaming and tracing support.\n", + "\n", "If we now look at LangSmith, we can see that the chain has two steps: first the language model is called, then the result of that is passed to the output parser. We can see the [LangSmith trace](https://smith.langchain.com/public/05bec1c1-fc51-4b2c-ab3b-4b63709e4462/r)" ] }, @@ -520,15 +517,22 @@ "source": [ "## Conclusion\n", "\n", - "That's it! In this tutorial we've walked through creating our first simple LLM application. We've learned how to work with language models, how to parse their outputs, how to create a prompt template, and how to get great observability into chains you create with LangSmith.\n", + "That's it! In this tutorial you've learned how to create your first simple LLM application. You've learned how to work with language models, how to parse their outputs, how to create a prompt template, chaining them together with LCEL, and how to get great observability into chains you create with LangSmith.\n", "\n", "This just scratches the surface of what you will want to learn to become a proficient AI Engineer. Luckily - we've got a lot of other resources!\n", "\n", - "For more in-depth tutorials, check out out [Tutorials](/docs/tutorials) section.\n", + "For further reading on the core concepts of LangChain, we've got detailed [Conceptual Guides](/docs/concepts).\n", + "\n", + "If you have more specific questions on these concepts, check out the following sections of the how-to guides:\n", + "\n", + "- [LangChain Expression Language (LCEL)](/docs/how_to/#langchain-expression-language)\n", + "- [Prompt templates](/docs/how_to/#prompt-templates)\n", + "- [Chat models](/docs/how_to/#chat-models)\n", + "- [Output parsers](/docs/how_to/#output-parsers)\n", "\n", - "If you have specific questions on how to accomplish particular tasks, see our [How-To Guides](/docs/how_to) section.\n", + "And the LangSmith docs:\n", "\n", - "For reading up on the core concepts of LangChain, we've got detailed [Conceptual Guides](/docs/concepts)" + "- [LangSmith](https://docs.smith.langchain.com)" ] } ], diff --git a/docs/core_docs/static/robots.txt b/docs/core_docs/static/robots.txt new file mode 100644 index 000000000000..6c90e23469be --- /dev/null +++ b/docs/core_docs/static/robots.txt @@ -0,0 +1,3 @@ +User-agent: * + +Sitemap: https://js.langchain.com/sitemap.xml/ \ No newline at end of file diff --git a/docs/core_docs/vercel.json b/docs/core_docs/vercel.json index b808cbe3e215..6334d323ca32 100644 --- a/docs/core_docs/vercel.json +++ b/docs/core_docs/vercel.json @@ -6,6 +6,14 @@ { "source": "/v0.1/:path(.*/?)*", "destination": "https://langchainjs-v01.vercel.app/v0.1/:path*" + }, + { + "source": "/robots.txt(/?)", + "destination": "/v0.2/robots.txt/" + }, + { + "source": "/sitemap.xml(/?)", + "destination": "/v0.2/sitemap.xml/" } ], "redirects": [ @@ -44,6 +52,14 @@ { "source": "/docs/how_to/tool_calls_multi_modal(/?)", "destination": "/docs/how_to/multimodal_inputs/" + }, + { + "source": "/docs/langgraph(/?)", + "destination": "https://langchain-ai.github.io/langgraphjs/" + }, + { + "source": "/docs/langsmith(/?)", + "destination": "https://docs.smith.langchain.com/" } ] } diff --git a/environment_tests/test-exports-bun/src/entrypoints.js b/environment_tests/test-exports-bun/src/entrypoints.js index 068747384a8d..0127a63d1c08 100644 --- a/environment_tests/test-exports-bun/src/entrypoints.js +++ b/environment_tests/test-exports-bun/src/entrypoints.js @@ -37,6 +37,7 @@ export * from "langchain/callbacks"; export * from "langchain/output_parsers"; export * from "langchain/retrievers/contextual_compression"; export * from "langchain/retrievers/document_compressors"; +export * from "langchain/retrievers/ensemble"; export * from "langchain/retrievers/multi_query"; export * from "langchain/retrievers/multi_vector"; export * from "langchain/retrievers/parent_document"; diff --git a/environment_tests/test-exports-cf/src/entrypoints.js b/environment_tests/test-exports-cf/src/entrypoints.js index 068747384a8d..0127a63d1c08 100644 --- a/environment_tests/test-exports-cf/src/entrypoints.js +++ b/environment_tests/test-exports-cf/src/entrypoints.js @@ -37,6 +37,7 @@ export * from "langchain/callbacks"; export * from "langchain/output_parsers"; export * from "langchain/retrievers/contextual_compression"; export * from "langchain/retrievers/document_compressors"; +export * from "langchain/retrievers/ensemble"; export * from "langchain/retrievers/multi_query"; export * from "langchain/retrievers/multi_vector"; export * from "langchain/retrievers/parent_document"; diff --git a/environment_tests/test-exports-cjs/src/entrypoints.js b/environment_tests/test-exports-cjs/src/entrypoints.js index d081d45f6aeb..5f9a19db39f2 100644 --- a/environment_tests/test-exports-cjs/src/entrypoints.js +++ b/environment_tests/test-exports-cjs/src/entrypoints.js @@ -37,6 +37,7 @@ const callbacks = require("langchain/callbacks"); const output_parsers = require("langchain/output_parsers"); const retrievers_contextual_compression = require("langchain/retrievers/contextual_compression"); const retrievers_document_compressors = require("langchain/retrievers/document_compressors"); +const retrievers_ensemble = require("langchain/retrievers/ensemble"); const retrievers_multi_query = require("langchain/retrievers/multi_query"); const retrievers_multi_vector = require("langchain/retrievers/multi_vector"); const retrievers_parent_document = require("langchain/retrievers/parent_document"); diff --git a/environment_tests/test-exports-esbuild/src/entrypoints.js b/environment_tests/test-exports-esbuild/src/entrypoints.js index 4b8bd265fff9..d3b76a743d8a 100644 --- a/environment_tests/test-exports-esbuild/src/entrypoints.js +++ b/environment_tests/test-exports-esbuild/src/entrypoints.js @@ -37,6 +37,7 @@ import * as callbacks from "langchain/callbacks"; import * as output_parsers from "langchain/output_parsers"; import * as retrievers_contextual_compression from "langchain/retrievers/contextual_compression"; import * as retrievers_document_compressors from "langchain/retrievers/document_compressors"; +import * as retrievers_ensemble from "langchain/retrievers/ensemble"; import * as retrievers_multi_query from "langchain/retrievers/multi_query"; import * as retrievers_multi_vector from "langchain/retrievers/multi_vector"; import * as retrievers_parent_document from "langchain/retrievers/parent_document"; diff --git a/environment_tests/test-exports-esm/src/entrypoints.js b/environment_tests/test-exports-esm/src/entrypoints.js index 4b8bd265fff9..d3b76a743d8a 100644 --- a/environment_tests/test-exports-esm/src/entrypoints.js +++ b/environment_tests/test-exports-esm/src/entrypoints.js @@ -37,6 +37,7 @@ import * as callbacks from "langchain/callbacks"; import * as output_parsers from "langchain/output_parsers"; import * as retrievers_contextual_compression from "langchain/retrievers/contextual_compression"; import * as retrievers_document_compressors from "langchain/retrievers/document_compressors"; +import * as retrievers_ensemble from "langchain/retrievers/ensemble"; import * as retrievers_multi_query from "langchain/retrievers/multi_query"; import * as retrievers_multi_vector from "langchain/retrievers/multi_vector"; import * as retrievers_parent_document from "langchain/retrievers/parent_document"; diff --git a/environment_tests/test-exports-vercel/src/entrypoints.js b/environment_tests/test-exports-vercel/src/entrypoints.js index 068747384a8d..0127a63d1c08 100644 --- a/environment_tests/test-exports-vercel/src/entrypoints.js +++ b/environment_tests/test-exports-vercel/src/entrypoints.js @@ -37,6 +37,7 @@ export * from "langchain/callbacks"; export * from "langchain/output_parsers"; export * from "langchain/retrievers/contextual_compression"; export * from "langchain/retrievers/document_compressors"; +export * from "langchain/retrievers/ensemble"; export * from "langchain/retrievers/multi_query"; export * from "langchain/retrievers/multi_vector"; export * from "langchain/retrievers/parent_document"; diff --git a/environment_tests/test-exports-vite/src/entrypoints.js b/environment_tests/test-exports-vite/src/entrypoints.js index 068747384a8d..0127a63d1c08 100644 --- a/environment_tests/test-exports-vite/src/entrypoints.js +++ b/environment_tests/test-exports-vite/src/entrypoints.js @@ -37,6 +37,7 @@ export * from "langchain/callbacks"; export * from "langchain/output_parsers"; export * from "langchain/retrievers/contextual_compression"; export * from "langchain/retrievers/document_compressors"; +export * from "langchain/retrievers/ensemble"; export * from "langchain/retrievers/multi_query"; export * from "langchain/retrievers/multi_vector"; export * from "langchain/retrievers/parent_document"; diff --git a/examples/src/retrievers/ensemble_retriever.ts b/examples/src/retrievers/ensemble_retriever.ts new file mode 100644 index 000000000000..e8fc3a15c874 --- /dev/null +++ b/examples/src/retrievers/ensemble_retriever.ts @@ -0,0 +1,67 @@ +import { EnsembleRetriever } from "langchain/retrievers/ensemble"; +import { MemoryVectorStore } from "langchain/vectorstores/memory"; +import { OpenAIEmbeddings } from "@langchain/openai"; +import { BaseRetriever, BaseRetrieverInput } from "@langchain/core/retrievers"; +import { Document } from "@langchain/core/documents"; + +class SimpleCustomRetriever extends BaseRetriever { + lc_namespace = []; + + documents: Document[]; + + constructor(fields: { documents: Document[] } & BaseRetrieverInput) { + super(fields); + this.documents = fields.documents; + } + + async _getRelevantDocuments(query: string): Promise { + return this.documents.filter((document) => + document.pageContent.includes(query) + ); + } +} + +const docs1 = [ + new Document({ pageContent: "I like apples", metadata: { source: 1 } }), + new Document({ pageContent: "I like oranges", metadata: { source: 1 } }), + new Document({ + pageContent: "apples and oranges are fruits", + metadata: { source: 1 }, + }), +]; + +const keywordRetriever = new SimpleCustomRetriever({ documents: docs1 }); + +const docs2 = [ + new Document({ pageContent: "You like apples", metadata: { source: 2 } }), + new Document({ pageContent: "You like oranges", metadata: { source: 2 } }), +]; + +const vectorstore = await MemoryVectorStore.fromDocuments( + docs2, + new OpenAIEmbeddings() +); + +const vectorstoreRetriever = vectorstore.asRetriever(); + +const retriever = new EnsembleRetriever({ + retrievers: [vectorstoreRetriever, keywordRetriever], + weights: [0.5, 0.5], +}); + +const query = "apples"; +const retrievedDocs = await retriever.invoke(query); + +console.log(retrievedDocs); + +/* + [ + Document { pageContent: 'You like apples', metadata: { source: 2 } }, + Document { pageContent: 'I like apples', metadata: { source: 1 } }, + Document { pageContent: 'You like oranges', metadata: { source: 2 } }, + Document { + pageContent: 'apples and oranges are fruits', + metadata: { source: 1 } + } + ] +*/ diff --git a/langchain-core/package.json b/langchain-core/package.json index b246974d0636..abc017dc35ce 100644 --- a/langchain-core/package.json +++ b/langchain-core/package.json @@ -1,6 +1,6 @@ { "name": "@langchain/core", - "version": "0.2.5", + "version": "0.2.6", "description": "Core LangChain.js abstractions and schemas", "type": "module", "engines": { diff --git a/langchain-core/src/callbacks/promises.ts b/langchain-core/src/callbacks/promises.ts index ddab6e5339ea..3484a0b4a522 100644 --- a/langchain-core/src/callbacks/promises.ts +++ b/langchain-core/src/callbacks/promises.ts @@ -17,7 +17,7 @@ function createQueue() { /** * Consume a promise, either adding it to the queue or waiting for it to resolve - * @param promise Promise to consume + * @param promiseFn Promise to consume * @param wait Whether to wait for the promise to resolve or resolve immediately */ export async function consumeCallback( diff --git a/langchain-core/src/language_models/chat_models.ts b/langchain-core/src/language_models/chat_models.ts index 64924bbebd47..d78bb04b2e0e 100644 --- a/langchain-core/src/language_models/chat_models.ts +++ b/langchain-core/src/language_models/chat_models.ts @@ -263,7 +263,7 @@ export abstract class BaseChatModel< } } - protected getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { return { ls_model_type: "chat", ls_stop: options.stop, diff --git a/langchain-core/src/prompts/chat.ts b/langchain-core/src/prompts/chat.ts index d1fbbabc6cc0..66c5d6c61839 100644 --- a/langchain-core/src/prompts/chat.ts +++ b/langchain-core/src/prompts/chat.ts @@ -32,7 +32,12 @@ import { ExtractedFStringParams, } from "./prompt.js"; import { ImagePromptTemplate } from "./image.js"; -import { TemplateFormat, parseFString } from "./template.js"; +import { + ParsedTemplateNode, + TemplateFormat, + parseFString, + parseMustache, +} from "./template.js"; /** * Abstract class that serves as a base for creating message prompt @@ -495,13 +500,19 @@ class _StringImageMessagePromptTemplate< } else if (typeof item.text === "string") { text = item.text ?? ""; } - prompt.push(PromptTemplate.fromTemplate(text)); + prompt.push(PromptTemplate.fromTemplate(text, additionalOptions)); } else if (typeof item === "object" && "image_url" in item) { let imgTemplate = item.image_url ?? ""; let imgTemplateObject: ImagePromptTemplate; let inputVariables: string[] = []; if (typeof imgTemplate === "string") { - const parsedTemplate = parseFString(imgTemplate); + let parsedTemplate: ParsedTemplateNode[]; + if (additionalOptions?.templateFormat === "mustache") { + parsedTemplate = parseMustache(imgTemplate); + } else { + parsedTemplate = parseFString(imgTemplate); + } + const variables = parsedTemplate.flatMap((item) => item.type === "variable" ? [item.name] : [] ); @@ -524,7 +535,13 @@ class _StringImageMessagePromptTemplate< }); } else if (typeof imgTemplate === "object") { if ("url" in imgTemplate) { - const parsedTemplate = parseFString(imgTemplate.url); + let parsedTemplate: ParsedTemplateNode[]; + if (additionalOptions?.templateFormat === "mustache") { + parsedTemplate = parseMustache(imgTemplate.url); + } else { + parsedTemplate = parseFString(imgTemplate.url); + } + inputVariables = parsedTemplate.flatMap((item) => item.type === "variable" ? [item.name] : [] ); @@ -913,7 +930,12 @@ export class ChatPromptTemplate< imageUrl = item.image_url.url; } - const promptTemplatePlaceholder = PromptTemplate.fromTemplate(imageUrl); + const promptTemplatePlaceholder = PromptTemplate.fromTemplate( + imageUrl, + { + templateFormat: this.templateFormat, + } + ); const formattedUrl = await promptTemplatePlaceholder.format( inputValues ); diff --git a/langchain-core/src/prompts/tests/chat.mustache.test.ts b/langchain-core/src/prompts/tests/chat.mustache.test.ts index 513814b0de93..ecac9e954634 100644 --- a/langchain-core/src/prompts/tests/chat.mustache.test.ts +++ b/langchain-core/src/prompts/tests/chat.mustache.test.ts @@ -2,7 +2,7 @@ import { test, expect } from "@jest/globals"; import { AIMessage } from "../../messages/ai.js"; import { HumanMessage } from "../../messages/human.js"; import { SystemMessage } from "../../messages/system.js"; -import { ChatPromptTemplate } from "../chat.js"; +import { ChatPromptTemplate, HumanMessagePromptTemplate } from "../chat.js"; test("Test creating a chat prompt template from role string messages", async () => { const template = ChatPromptTemplate.fromMessages( @@ -67,3 +67,53 @@ test("Ignores f-string inputs input variables with repeats.", async () => { new HumanMessage("This {bar} is a {foo} test {foo}."), ]); }); + +test("Mustache template with image and chat prompts inside one template (fromMessages)", async () => { + const template = ChatPromptTemplate.fromMessages( + [ + [ + "human", + [ + { + type: "image_url", + image_url: "{{image_url}}", + }, + { + type: "text", + text: "{{other_var}}", + }, + ], + ], + ["human", "hello {{name}}"], + ], + { + templateFormat: "mustache", + } + ); + + expect(template.inputVariables.sort()).toEqual([ + "image_url", + "name", + "other_var", + ]); +}); + +test("Mustache image template with nested URL and chat prompts HumanMessagePromptTemplate.fromTemplate", async () => { + const template = HumanMessagePromptTemplate.fromTemplate( + [ + { + text: "{{name}}", + }, + { + image_url: { + url: "{{image_url}}", + }, + }, + ], + { + templateFormat: "mustache", + } + ); + + expect(template.inputVariables.sort()).toEqual(["image_url", "name"]); +}); diff --git a/langchain/.gitignore b/langchain/.gitignore index 694b69c6f9d8..768344a3b89f 100644 --- a/langchain/.gitignore +++ b/langchain/.gitignore @@ -358,6 +358,10 @@ retrievers/document_compressors.cjs retrievers/document_compressors.js retrievers/document_compressors.d.ts retrievers/document_compressors.d.cts +retrievers/ensemble.cjs +retrievers/ensemble.js +retrievers/ensemble.d.ts +retrievers/ensemble.d.cts retrievers/multi_query.cjs retrievers/multi_query.js retrievers/multi_query.d.ts diff --git a/langchain/langchain.config.js b/langchain/langchain.config.js index 1cd47cf5d1e8..5700b59dd7bc 100644 --- a/langchain/langchain.config.js +++ b/langchain/langchain.config.js @@ -143,6 +143,7 @@ export const config = { // retrievers "retrievers/contextual_compression": "retrievers/contextual_compression", "retrievers/document_compressors": "retrievers/document_compressors/index", + "retrievers/ensemble": "retrievers/ensemble", "retrievers/multi_query": "retrievers/multi_query", "retrievers/multi_vector": "retrievers/multi_vector", "retrievers/parent_document": "retrievers/parent_document", diff --git a/langchain/package.json b/langchain/package.json index 5510d32f70ed..ad7e4afd06b6 100644 --- a/langchain/package.json +++ b/langchain/package.json @@ -1,6 +1,6 @@ { "name": "langchain", - "version": "0.2.4", + "version": "0.2.5", "description": "Typescript bindings for langchain", "type": "module", "engines": { @@ -370,6 +370,10 @@ "retrievers/document_compressors.js", "retrievers/document_compressors.d.ts", "retrievers/document_compressors.d.cts", + "retrievers/ensemble.cjs", + "retrievers/ensemble.js", + "retrievers/ensemble.d.ts", + "retrievers/ensemble.d.cts", "retrievers/multi_query.cjs", "retrievers/multi_query.js", "retrievers/multi_query.d.ts", @@ -1725,6 +1729,15 @@ "import": "./retrievers/document_compressors.js", "require": "./retrievers/document_compressors.cjs" }, + "./retrievers/ensemble": { + "types": { + "import": "./retrievers/ensemble.d.ts", + "require": "./retrievers/ensemble.d.cts", + "default": "./retrievers/ensemble.d.ts" + }, + "import": "./retrievers/ensemble.js", + "require": "./retrievers/ensemble.cjs" + }, "./retrievers/multi_query": { "types": { "import": "./retrievers/multi_query.d.ts", diff --git a/langchain/src/load/import_map.ts b/langchain/src/load/import_map.ts index 2ac351c8fdba..115793fcac92 100644 --- a/langchain/src/load/import_map.ts +++ b/langchain/src/load/import_map.ts @@ -33,6 +33,7 @@ export * as callbacks from "../callbacks/index.js"; export * as output_parsers from "../output_parsers/index.js"; export * as retrievers__contextual_compression from "../retrievers/contextual_compression.js"; export * as retrievers__document_compressors from "../retrievers/document_compressors/index.js"; +export * as retrievers__ensemble from "../retrievers/ensemble.js"; export * as retrievers__multi_query from "../retrievers/multi_query.js"; export * as retrievers__multi_vector from "../retrievers/multi_vector.js"; export * as retrievers__parent_document from "../retrievers/parent_document.js"; diff --git a/langchain/src/retrievers/ensemble.ts b/langchain/src/retrievers/ensemble.ts new file mode 100644 index 000000000000..606b1cf79c22 --- /dev/null +++ b/langchain/src/retrievers/ensemble.ts @@ -0,0 +1,119 @@ +import { BaseRetriever, BaseRetrieverInput } from "@langchain/core/retrievers"; +import { Document, DocumentInterface } from "@langchain/core/documents"; +import { CallbackManagerForRetrieverRun } from "@langchain/core/callbacks/manager"; + +export interface EnsembleRetrieverInput extends BaseRetrieverInput { + /** A list of retrievers to ensemble. */ + retrievers: BaseRetriever[]; + /** + * A list of weights corresponding to the retrievers. Defaults to equal + * weighting for all retrievers. + */ + weights?: number[]; + /** + * A constant added to the rank, controlling the balance between the importance + * of high-ranked items and the consideration given to lower-ranked items. + * Default is 60. + */ + c?: number; +} + +/** + * Ensemble retriever that aggregates and orders the results of + * multiple retrievers by using weighted Reciprocal Rank Fusion. + */ +export class EnsembleRetriever extends BaseRetriever { + static lc_name() { + return "EnsembleRetriever"; + } + + lc_namespace = ["langchain", "retrievers", "ensemble_retriever"]; + + retrievers: BaseRetriever[]; + + weights: number[]; + + c = 60; + + constructor(args: EnsembleRetrieverInput) { + super(args); + this.retrievers = args.retrievers; + this.weights = + args.weights || + new Array(args.retrievers.length).fill(1 / args.retrievers.length); + this.c = args.c || 60; + } + + async _getRelevantDocuments( + query: string, + runManager?: CallbackManagerForRetrieverRun + ) { + return this._rankFusion(query, runManager); + } + + async _rankFusion( + query: string, + runManager?: CallbackManagerForRetrieverRun + ) { + const retrieverDocs = await Promise.all( + this.retrievers.map((retriever, i) => + retriever.invoke(query, { + callbacks: runManager?.getChild(`retriever_${i + 1}`), + }) + ) + ); + + const fusedDocs = await this._weightedReciprocalRank(retrieverDocs); + return fusedDocs; + } + + async _weightedReciprocalRank(docList: DocumentInterface[][]) { + if (docList.length !== this.weights.length) { + throw new Error( + "Number of retrieved document lists must be equal to the number of weights." + ); + } + + const rrfScoreDict = docList.reduce( + (rffScore: Record, retrieverDoc, idx) => { + let rank = 1; + const weight = this.weights[idx]; + while (rank <= retrieverDoc.length) { + const { pageContent } = retrieverDoc[rank - 1]; + if (!rffScore[pageContent]) { + // eslint-disable-next-line no-param-reassign + rffScore[pageContent] = 0; + } + // eslint-disable-next-line no-param-reassign + rffScore[pageContent] += weight / (rank + this.c); + rank += 1; + } + + return rffScore; + }, + {} + ); + + const uniqueDocs = this._uniqueUnion(docList.flat()); + const sortedDocs = Array.from(uniqueDocs).sort( + (a, b) => rrfScoreDict[b.pageContent] - rrfScoreDict[a.pageContent] + ); + + return sortedDocs; + } + + private _uniqueUnion(documents: Document[]): Document[] { + const documentSet = new Set(); + const result = []; + + for (const doc of documents) { + const key = doc.pageContent; + if (!documentSet.has(key)) { + documentSet.add(key); + result.push(doc); + } + } + + return result; + } +} diff --git a/langchain/src/retrievers/tests/ensemble_retriever.int.test.ts b/langchain/src/retrievers/tests/ensemble_retriever.int.test.ts new file mode 100644 index 000000000000..781dfb859321 --- /dev/null +++ b/langchain/src/retrievers/tests/ensemble_retriever.int.test.ts @@ -0,0 +1,104 @@ +import { expect, test } from "@jest/globals"; +import { CohereEmbeddings } from "@langchain/cohere"; +import { MemoryVectorStore } from "../../vectorstores/memory.js"; +import { EnsembleRetriever } from "../ensemble.js"; + +test("Should work with a question input", async () => { + const vectorstore = await MemoryVectorStore.fromTexts( + [ + "Buildings are made out of brick", + "Buildings are made out of wood", + "Buildings are made out of stone", + "Cars are made out of metal", + "Cars are made out of plastic", + "mitochondria is the powerhouse of the cell", + "mitochondria is made of lipids", + ], + [{ id: 1 }, { id: 2 }, { id: 3 }, { id: 4 }, { id: 5 }], + new CohereEmbeddings() + ); + const retriever = new EnsembleRetriever({ + retrievers: [vectorstore.asRetriever()], + }); + + const query = "What are mitochondria made of?"; + const retrievedDocs = await retriever.invoke(query); + expect(retrievedDocs[0].pageContent).toContain("mitochondria"); +}); + +test("Should work with multiple retriever", async () => { + const vectorstore = await MemoryVectorStore.fromTexts( + [ + "Buildings are made out of brick", + "Buildings are made out of wood", + "Buildings are made out of stone", + "Cars are made out of metal", + "Cars are made out of plastic", + "mitochondria is the powerhouse of the cell", + "mitochondria is made of lipids", + ], + [{ id: 1 }, { id: 2 }, { id: 3 }, { id: 4 }, { id: 5 }], + new CohereEmbeddings() + ); + const vectorstore2 = await MemoryVectorStore.fromTexts( + [ + "Buildings are made out of brick", + "Buildings are made out of wood", + "Buildings are made out of stone", + "Cars are made out of metal", + "Cars are made out of plastic", + "mitochondria is the powerhouse of the cell", + "mitochondria is made of lipids", + ], + [{ id: 6 }, { id: 7 }, { id: 8 }, { id: 9 }, { id: 10 }], + new CohereEmbeddings() + ); + const retriever = new EnsembleRetriever({ + retrievers: [vectorstore.asRetriever(), vectorstore2.asRetriever()], + }); + + const query = "cars"; + const retrievedDocs = await retriever.invoke(query); + expect( + retrievedDocs.filter((item) => item.pageContent.includes("Cars")).length + ).toBe(2); +}); + +test("Should work with weights", async () => { + const vectorstore = await MemoryVectorStore.fromTexts( + [ + "Buildings are made out of brick", + "Buildings are made out of wood", + "Buildings are made out of stone", + "Cars are made out of metal", + "Cars are made out of plastic", + "mitochondria is the powerhouse of the cell", + "mitochondria is made of lipids", + ], + [{ id: 1 }, { id: 2 }, { id: 3 }, { id: 4 }, { id: 5 }], + new CohereEmbeddings() + ); + const vectorstore2 = await MemoryVectorStore.fromTexts( + [ + "Buildings are made out of brick", + "Buildings are made out of wood", + "Buildings are made out of stone", + "Cars are made out of metal", + "Cars are made out of plastic", + "mitochondria is the powerhouse of the cell", + "mitochondria is made of lipids", + ], + [{ id: 6 }, { id: 7 }, { id: 8 }, { id: 9 }, { id: 10 }], + new CohereEmbeddings() + ); + const retriever = new EnsembleRetriever({ + retrievers: [vectorstore.asRetriever(), vectorstore2.asRetriever()], + weights: [0.5, 0.9], + }); + + const query = "cars"; + const retrievedDocs = await retriever.invoke(query); + expect( + retrievedDocs.filter((item) => item.pageContent.includes("Cars")).length + ).toBe(2); +}); diff --git a/libs/langchain-anthropic/package.json b/libs/langchain-anthropic/package.json index 65f158e73501..3e61dd9b456e 100644 --- a/libs/langchain-anthropic/package.json +++ b/libs/langchain-anthropic/package.json @@ -16,26 +16,21 @@ "scripts": { "build": "yarn turbo:command build:internal --filter=@langchain/anthropic", "build:internal": "yarn lc-build:v2 --create-entrypoints --pre --tree-shaking --gen-maps", - "build:deps": "yarn run turbo:command build --filter=@langchain/core", - "build:esm": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist/", - "build:cjs": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist-cjs/ -p tsconfig.cjs.json && yarn move-cjs-to-dist && rimraf dist-cjs", - "build:watch": "yarn create-entrypoints && tsc --outDir dist/ --watch", - "build:scripts": "yarn create-entrypoints && yarn check-tree-shaking", "lint:eslint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/", "lint:dpdm": "dpdm --exit-code circular:1 --no-warning --no-tree src/*.ts src/**/*.ts", "lint": "yarn lint:eslint && yarn lint:dpdm", "lint:fix": "yarn lint:eslint --fix && yarn lint:dpdm", "clean": "rm -rf .turbo dist/", "prepack": "yarn build", - "test": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%", - "test:watch": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", - "test:single": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", - "test:int": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test": "NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%", + "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", + "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", + "test:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:unit": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard": "yarn test:standard:unit && yarn test:standard:int", "format": "prettier --config .prettierrc --write \"src\"", - "format:check": "prettier --config .prettierrc --check \"src\"", - "move-cjs-to-dist": "yarn lc-build --config ./langchain.config.js --move-cjs-dist", - "create-entrypoints": "yarn lc-build --config ./langchain.config.js --create-entrypoints", - "check-tree-shaking": "yarn lc-build --config ./langchain.config.js --tree-shaking" + "format:check": "prettier --config .prettierrc --check \"src\"" }, "author": "LangChain", "license": "MIT", @@ -50,6 +45,7 @@ "@jest/globals": "^29.5.0", "@langchain/community": "workspace:*", "@langchain/scripts": "~0.0.14", + "@langchain/standard-tests": "workspace:*", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "dpdm": "^3.12.0", diff --git a/libs/langchain-anthropic/src/chat_models.ts b/libs/langchain-anthropic/src/chat_models.ts index 9b270ad37752..aba95cee3a00 100644 --- a/libs/langchain-anthropic/src/chat_models.ts +++ b/libs/langchain-anthropic/src/chat_models.ts @@ -67,7 +67,7 @@ type AnthropicToolChoice = } | "any" | "auto"; -interface ChatAnthropicCallOptions extends BaseLanguageModelCallOptions { +export interface ChatAnthropicCallOptions extends BaseLanguageModelCallOptions { tools?: (StructuredToolInterface | AnthropicTool)[]; /** * Whether or not to specify what tool the model should use @@ -518,7 +518,7 @@ export class ChatAnthropicMessages< this.clientOptions = fields?.clientOptions ?? {}; } - protected getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { const params = this.invocationParams(options); return { ls_provider: "openai", diff --git a/libs/langchain-anthropic/src/tests/chat_models.standard.int.test.ts b/libs/langchain-anthropic/src/tests/chat_models.standard.int.test.ts new file mode 100644 index 000000000000..45eff821ce9c --- /dev/null +++ b/libs/langchain-anthropic/src/tests/chat_models.standard.int.test.ts @@ -0,0 +1,39 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelIntegrationTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { ChatAnthropic, ChatAnthropicCallOptions } from "../chat_models.js"; + +class ChatAnthropicStandardIntegrationTests extends ChatModelIntegrationTests< + ChatAnthropicCallOptions, + AIMessageChunk +> { + constructor() { + if (!process.env.ANTHROPIC_API_KEY) { + throw new Error( + "ANTHROPIC_API_KEY must be set to run standard integration tests." + ); + } + super({ + Cls: ChatAnthropic, + chatModelHasToolCalling: true, + chatModelHasStructuredOutput: true, + constructorArgs: { + model: "claude-3-haiku-20240307", + }, + }); + } + + async testUsageMetadataStreaming() { + console.warn( + "Skipping testUsageMetadataStreaming, not implemented in ChatAnthropic." + ); + } +} + +const testClass = new ChatAnthropicStandardIntegrationTests(); + +test("ChatAnthropicStandardIntegrationTests", async () => { + const testResults = await testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-anthropic/src/tests/chat_models.standard.test.ts b/libs/langchain-anthropic/src/tests/chat_models.standard.test.ts new file mode 100644 index 000000000000..7ddefe42b8f4 --- /dev/null +++ b/libs/langchain-anthropic/src/tests/chat_models.standard.test.ts @@ -0,0 +1,39 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelUnitTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { ChatAnthropic, ChatAnthropicCallOptions } from "../chat_models.js"; + +class ChatAnthropicStandardUnitTests extends ChatModelUnitTests< + ChatAnthropicCallOptions, + AIMessageChunk +> { + constructor() { + super({ + Cls: ChatAnthropic, + chatModelHasToolCalling: true, + chatModelHasStructuredOutput: true, + constructorArgs: {}, + }); + // This must be set so method like `.bindTools` or `.withStructuredOutput` + // which we call after instantiating the model will work. + // (constructor will throw if API key is not set) + process.env.ANTHROPIC_API_KEY = "test"; + } + + testChatModelInitApiKey() { + // Unset the API key env var here so this test can properly check + // the API key class arg. + process.env.ANTHROPIC_API_KEY = ""; + super.testChatModelInitApiKey(); + // Re-set the API key env var here so other tests can run properly. + process.env.ANTHROPIC_API_KEY = "test"; + } +} + +const testClass = new ChatAnthropicStandardUnitTests(); + +test("ChatAnthropicStandardUnitTests", () => { + const testResults = testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-azure-openai/package.json b/libs/langchain-azure-openai/package.json index 17c6c15a5f7c..e7f498c05c18 100644 --- a/libs/langchain-azure-openai/package.json +++ b/libs/langchain-azure-openai/package.json @@ -15,26 +15,18 @@ "scripts": { "build": "yarn turbo:command build:internal --filter=@langchain/azure-openai", "build:internal": "yarn lc-build:v2 --create-entrypoints --pre --tree-shaking", - "build:deps": "yarn run turbo:command build --filter=@langchain/core", - "build:esm": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist/", - "build:cjs": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist-cjs/ -p tsconfig.cjs.json && yarn move-cjs-to-dist && rimraf dist-cjs", - "build:watch": "yarn create-entrypoints && tsc --outDir dist/ --watch", - "build:scripts": "yarn create-entrypoints && yarn check-tree-shaking", "lint:eslint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/", "lint:dpdm": "dpdm --exit-code circular:1 --no-warning --no-tree src/*.ts src/**/*.ts", "lint": "yarn lint:eslint && yarn lint:dpdm", "lint:fix": "yarn lint:eslint --fix && yarn lint:dpdm", "clean": "rm -rf .turbo dist/", "prepack": "yarn build", - "test": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%", - "test:watch": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", - "test:single": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", + "test": "NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%", + "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", + "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", "test:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", "format": "prettier --config .prettierrc --write \"src\"", - "format:check": "prettier --config .prettierrc --check \"src\"", - "move-cjs-to-dist": "yarn lc-build --config ./langchain.config.js --move-cjs-dist", - "create-entrypoints": "yarn lc-build --config ./langchain.config.js --create-entrypoints", - "check-tree-shaking": "yarn lc-build --config ./langchain.config.js --tree-shaking" + "format:check": "prettier --config .prettierrc --check \"src\"" }, "author": "LangChain", "license": "MIT", @@ -50,6 +42,7 @@ "@azure/identity": "^4.0.1", "@jest/globals": "^29.5.0", "@langchain/scripts": "~0.0.14", + "@langchain/standard-tests": "workspace:*", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "dpdm": "^3.12.0", diff --git a/libs/langchain-cloudflare/package.json b/libs/langchain-cloudflare/package.json index f6141d965bb5..8b5e6d9afefd 100644 --- a/libs/langchain-cloudflare/package.json +++ b/libs/langchain-cloudflare/package.json @@ -16,10 +16,6 @@ "scripts": { "build": "yarn turbo:command build:internal --filter=@langchain/cloudflare", "build:internal": "yarn lc-build:v2 --create-entrypoints --pre --tree-shaking", - "build:esm": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist/ && rm -rf dist/tests dist/**/tests", - "build:cjs": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist-cjs/ -p tsconfig.cjs.json && yarn move-cjs-to-dist && rm -rf dist-cjs", - "build:watch": "yarn create-entrypoints && tsc --outDir dist/ --watch", - "build:scripts": "yarn create-entrypoints && yarn check-tree-shaking", "lint:eslint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/", "lint:dpdm": "dpdm --exit-code circular:1 --no-warning --no-tree src/*.ts src/**/*.ts", "lint": "yarn lint:eslint && yarn lint:dpdm", @@ -30,11 +26,11 @@ "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", "test:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:unit": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard": "yarn test:standard:unit && yarn test:standard:int", "format": "prettier --config .prettierrc --write \"src\"", - "format:check": "prettier --config .prettierrc --check \"src\"", - "move-cjs-to-dist": "yarn lc-build --config ./langchain.config.js --move-cjs-dist", - "create-entrypoints": "yarn lc-build --config ./langchain.config.js --create-entrypoints", - "check-tree-shaking": "yarn lc-build --config ./langchain.config.js --tree-shaking" + "format:check": "prettier --config .prettierrc --check \"src\"" }, "author": "LangChain", "license": "MIT", @@ -47,6 +43,7 @@ "@cloudflare/workers-types": "^4.20231218.0", "@jest/globals": "^29.5.0", "@langchain/scripts": "~0.0.14", + "@langchain/standard-tests": "workspace:*", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "@tsconfig/recommended": "^1.0.3", diff --git a/libs/langchain-cloudflare/src/chat_models.ts b/libs/langchain-cloudflare/src/chat_models.ts index af4576d18654..bdae5020d6ba 100644 --- a/libs/langchain-cloudflare/src/chat_models.ts +++ b/libs/langchain-cloudflare/src/chat_models.ts @@ -1,4 +1,5 @@ import { + LangSmithParams, SimpleChatModel, type BaseChatModelParams, } from "@langchain/core/language_models/chat_models"; @@ -81,6 +82,15 @@ export class ChatCloudflareWorkersAI } } + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { + return { + ls_provider: "openai", + ls_model_name: this.model, + ls_model_type: "chat", + ls_stop: options.stop, + }; + } + get lc_secrets(): { [key: string]: string } | undefined { return { cloudflareApiToken: "CLOUDFLARE_API_TOKEN", diff --git a/libs/langchain-cloudflare/src/tests/chat_models.standard.int.test.ts b/libs/langchain-cloudflare/src/tests/chat_models.standard.int.test.ts new file mode 100644 index 000000000000..c49d3b7d009d --- /dev/null +++ b/libs/langchain-cloudflare/src/tests/chat_models.standard.int.test.ts @@ -0,0 +1,53 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelIntegrationTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { + ChatCloudflareWorkersAI, + ChatCloudflareWorkersAICallOptions, +} from "../chat_models.js"; + +class ChatCloudflareWorkersAIStandardIntegrationTests extends ChatModelIntegrationTests< + ChatCloudflareWorkersAICallOptions, + AIMessageChunk +> { + constructor() { + if ( + !process.env.CLOUDFLARE_ACCOUNT_ID || + !process.env.CLOUDFLARE_API_TOKEN + ) { + throw new Error( + "Skipping Cloudflare Workers AI integration tests because CLOUDFLARE_ACCOUNT_ID or CLOUDFLARE_API_TOKEN is not set" + ); + } + super({ + Cls: ChatCloudflareWorkersAI, + chatModelHasToolCalling: false, + chatModelHasStructuredOutput: false, + constructorArgs: {}, + }); + } + + async testUsageMetadataStreaming() { + this.skipTestMessage( + "testUsageMetadataStreaming", + "ChatCloudflareWorkersAI", + "Streaming tokens is not currently supported." + ); + } + + async testUsageMetadata() { + this.skipTestMessage( + "testUsageMetadata", + "ChatCloudflareWorkersAI", + "Usage metadata tokens is not currently supported." + ); + } +} + +const testClass = new ChatCloudflareWorkersAIStandardIntegrationTests(); + +test("ChatCloudflareWorkersAIStandardIntegrationTests", async () => { + const testResults = await testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-cloudflare/src/tests/chat_models.standard.test.ts b/libs/langchain-cloudflare/src/tests/chat_models.standard.test.ts new file mode 100644 index 000000000000..1faa95c782e2 --- /dev/null +++ b/libs/langchain-cloudflare/src/tests/chat_models.standard.test.ts @@ -0,0 +1,50 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelUnitTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { LangSmithParams } from "@langchain/core/language_models/chat_models"; +import { + ChatCloudflareWorkersAI, + ChatCloudflareWorkersAICallOptions, +} from "../chat_models.js"; + +class ChatCloudflareWorkersAIStandardUnitTests extends ChatModelUnitTests< + ChatCloudflareWorkersAICallOptions, + AIMessageChunk +> { + constructor() { + super({ + Cls: ChatCloudflareWorkersAI, + chatModelHasToolCalling: false, + chatModelHasStructuredOutput: false, + constructorArgs: {}, + }); + } + + testChatModelInitApiKey() { + this.skipTestMessage( + "testChatModelInitApiKey", + "ChatCloudflareWorkersAI", + this.multipleApiKeysRequiredMessage + ); + } + + expectedLsParams(): Partial { + console.warn( + "Overriding testStandardParams. ChatCloudflareWorkersAI does not support temperature or max tokens." + ); + return { + ls_provider: "string", + ls_model_name: "string", + ls_model_type: "chat", + ls_stop: ["Array"], + }; + } +} + +const testClass = new ChatCloudflareWorkersAIStandardUnitTests(); + +test("ChatCloudflareWorkersAIStandardUnitTests", () => { + const testResults = testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-cohere/package.json b/libs/langchain-cohere/package.json index 220b7f2f635f..6aac0d2cddd5 100644 --- a/libs/langchain-cohere/package.json +++ b/libs/langchain-cohere/package.json @@ -16,10 +16,6 @@ "scripts": { "build": "yarn turbo:command build:internal --filter=@langchain/cohere", "build:internal": "yarn lc-build:v2 --create-entrypoints --pre --tree-shaking", - "build:esm": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist/ && rm -rf dist/tests dist/**/tests", - "build:cjs": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist-cjs/ -p tsconfig.cjs.json && yarn move-cjs-to-dist && rm -rf dist-cjs", - "build:watch": "yarn create-entrypoints && tsc --outDir dist/ --watch", - "build:scripts": "yarn create-entrypoints && yarn check-tree-shaking", "lint:eslint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/", "lint:dpdm": "dpdm --exit-code circular:1 --no-warning --no-tree src/*.ts src/**/*.ts", "lint": "yarn lint:eslint && yarn lint:dpdm", @@ -30,11 +26,11 @@ "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", "test:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:unit": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard": "yarn test:standard:unit && yarn test:standard:int", "format": "prettier --config .prettierrc --write \"src\"", - "format:check": "prettier --config .prettierrc --check \"src\"", - "move-cjs-to-dist": "yarn lc-build --config ./langchain.config.js --move-cjs-dist", - "create-entrypoints": "yarn lc-build --config ./langchain.config.js --create-entrypoints", - "check-tree-shaking": "yarn lc-build --config ./langchain.config.js --tree-shaking" + "format:check": "prettier --config .prettierrc --check \"src\"" }, "author": "LangChain", "license": "MIT", @@ -45,6 +41,7 @@ "devDependencies": { "@jest/globals": "^29.5.0", "@langchain/scripts": "~0.0.14", + "@langchain/standard-tests": "workspace:*", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "@tsconfig/recommended": "^1.0.3", diff --git a/libs/langchain-cohere/src/chat_models.ts b/libs/langchain-cohere/src/chat_models.ts index 1aeca0175581..2179ec0bbc5d 100644 --- a/libs/langchain-cohere/src/chat_models.ts +++ b/libs/langchain-cohere/src/chat_models.ts @@ -55,7 +55,7 @@ interface TokenUsage { totalTokens?: number; } -interface CohereChatCallOptions +export interface CohereChatCallOptions extends BaseLanguageModelCallOptions, Partial>, Partial> {} @@ -146,7 +146,7 @@ export class ChatCohere< this.streaming = fields?.streaming ?? this.streaming; } - protected getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { const params = this.invocationParams(options); return { ls_provider: "cohere", diff --git a/libs/langchain-cohere/src/tests/chat_models.standard.int.test.ts b/libs/langchain-cohere/src/tests/chat_models.standard.int.test.ts new file mode 100644 index 000000000000..813aa00da412 --- /dev/null +++ b/libs/langchain-cohere/src/tests/chat_models.standard.int.test.ts @@ -0,0 +1,47 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelIntegrationTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { ChatCohere, CohereChatCallOptions } from "../chat_models.js"; + +class ChatCohereStandardIntegrationTests extends ChatModelIntegrationTests< + CohereChatCallOptions, + AIMessageChunk +> { + constructor() { + if (!process.env.COHERE_API_KEY) { + throw new Error( + "Can not run Cohere integration tests because COHERE_API_KEY is not set" + ); + } + super({ + Cls: ChatCohere, + chatModelHasToolCalling: false, + chatModelHasStructuredOutput: false, + constructorArgs: {}, + }); + } + + async testUsageMetadataStreaming() { + this.skipTestMessage( + "testUsageMetadataStreaming", + "ChatCohere", + "Streaming tokens is not currently supported." + ); + } + + async testUsageMetadata() { + this.skipTestMessage( + "testUsageMetadata", + "ChatCohere", + "Usage metadata tokens is not currently supported." + ); + } +} + +const testClass = new ChatCohereStandardIntegrationTests(); + +test("ChatCohereStandardIntegrationTests", async () => { + const testResults = await testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-cohere/src/tests/chat_models.standard.test.ts b/libs/langchain-cohere/src/tests/chat_models.standard.test.ts new file mode 100644 index 000000000000..dbfc2813ae83 --- /dev/null +++ b/libs/langchain-cohere/src/tests/chat_models.standard.test.ts @@ -0,0 +1,39 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelUnitTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { ChatCohere, CohereChatCallOptions } from "../chat_models.js"; + +class ChatCohereStandardUnitTests extends ChatModelUnitTests< + CohereChatCallOptions, + AIMessageChunk +> { + constructor() { + super({ + Cls: ChatCohere, + chatModelHasToolCalling: false, + chatModelHasStructuredOutput: false, + constructorArgs: {}, + }); + // This must be set so method like `.bindTools` or `.withStructuredOutput` + // which we call after instantiating the model will work. + // (constructor will throw if API key is not set) + process.env.COHERE_API_KEY = "test"; + } + + testChatModelInitApiKey() { + // Unset the API key env var here so this test can properly check + // the API key class arg. + process.env.COHERE_API_KEY = ""; + super.testChatModelInitApiKey(); + // Re-set the API key env var here so other tests can run properly. + process.env.COHERE_API_KEY = "test"; + } +} + +const testClass = new ChatCohereStandardUnitTests(); + +test("ChatCohereStandardUnitTests", () => { + const testResults = testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-community/package.json b/libs/langchain-community/package.json index 255f7d463702..42ed9888a5ad 100644 --- a/libs/langchain-community/package.json +++ b/libs/langchain-community/package.json @@ -1,6 +1,6 @@ { "name": "@langchain/community", - "version": "0.2.5", + "version": "0.2.6", "description": "Third-party integrations for LangChain.js", "type": "module", "engines": { @@ -16,26 +16,21 @@ "scripts": { "build": "yarn turbo:command build:internal --filter=@langchain/community", "build:internal": "yarn lc-build:v2 --create-entrypoints --pre --tree-shaking --gen-maps", - "build:deps": "yarn run turbo:command build --filter=@langchain/core --filter=@langchain/openai --filter=langchain --filter=@langchain/anthropic --concurrency=1", - "build:esm": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist/ && rm -rf dist/tests dist/**/tests", - "build:cjs": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist-cjs/ -p tsconfig.cjs.json && yarn move-cjs-to-dist && rm -rf dist-cjs", - "build:watch": "yarn create-entrypoints && tsc --outDir dist/ --watch", - "build:scripts": "yarn create-entrypoints && yarn check-tree-shaking", "lint:eslint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/", "lint:dpdm": "dpdm --exit-code circular:1 --no-warning --no-tree src/*.ts src/**/*.ts", "lint": "yarn lint:eslint && yarn lint:dpdm", "lint:fix": "yarn lint:eslint --fix && yarn lint:dpdm", "clean": "rm -rf .turbo dist/", "prepack": "yarn build", - "test": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%", - "test:watch": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", - "test:single": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", - "test:integration": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test": "NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%", + "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", + "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", + "test:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:unit": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard": "yarn test:standard:unit && yarn test:standard:int", "format": "prettier --config .prettierrc --write \"src\"", - "format:check": "prettier --config .prettierrc --check \"src\"", - "move-cjs-to-dist": "yarn lc-build --config ./langchain.config.js --move-cjs-dist", - "create-entrypoints": "yarn lc-build --config ./langchain.config.js --create-entrypoints", - "check-tree-shaking": "yarn lc-build --config ./langchain.config.js --tree-shaking" + "format:check": "prettier --config .prettierrc --check \"src\"" }, "author": "LangChain", "license": "MIT", @@ -84,9 +79,10 @@ "@huggingface/inference": "^2.6.4", "@jest/globals": "^29.5.0", "@langchain/scripts": "~0.0.14", + "@langchain/standard-tests": "workspace:*", "@layerup/layerup-security": "^1.5.12", "@mendable/firecrawl-js": "^0.0.13", - "@mlc-ai/web-llm": "^0.2.35", + "@mlc-ai/web-llm": "^0.2.40", "@mozilla/readability": "^0.4.4", "@neondatabase/serverless": "^0.9.1", "@notionhq/client": "^2.2.10", @@ -245,7 +241,7 @@ "@huggingface/inference": "^2.6.4", "@layerup/layerup-security": "^1.5.12", "@mendable/firecrawl-js": "^0.0.13", - "@mlc-ai/web-llm": "^0.2.35", + "@mlc-ai/web-llm": "^0.2.40", "@mozilla/readability": "*", "@neondatabase/serverless": "*", "@notionhq/client": "^2.2.10", diff --git a/libs/langchain-community/src/chat_models/bedrock/web.ts b/libs/langchain-community/src/chat_models/bedrock/web.ts index abf10fe0c208..1b6e0757772a 100644 --- a/libs/langchain-community/src/chat_models/bedrock/web.ts +++ b/libs/langchain-community/src/chat_models/bedrock/web.ts @@ -8,6 +8,7 @@ import { CallbackManagerForLLMRun } from "@langchain/core/callbacks/manager"; import { type BaseChatModelParams, BaseChatModel, + LangSmithParams, } from "@langchain/core/language_models/chat_models"; import { getEnvironmentVariable } from "@langchain/core/utils/env"; import { @@ -97,14 +98,92 @@ export function convertMessagesToPrompt( * Services (AWS). It uses AWS credentials for authentication and can be * configured with various parameters such as the model to use, the AWS * region, and the maximum number of tokens to generate. + * + * The `BedrockChat` class supports both synchronous and asynchronous interactions with the model, + * allowing for streaming responses and handling new token callbacks. It can be configured with + * optional parameters like temperature, stop sequences, and guardrail settings for enhanced control + * over the generated responses. + * * @example * ```typescript - * const model = new BedrockChat({ - * model: "anthropic.claude-v2", - * region: "us-east-1", - * }); - * const res = await model.invoke([{ content: "Tell me a joke" }]); - * console.log(res); + * import { BedrockChat } from 'path-to-your-bedrock-chat-module'; + * import { HumanMessage } from '@langchain/core/messages'; + * + * async function run() { + * // Instantiate the BedrockChat model with the desired configuration + * const model = new BedrockChat({ + * model: "anthropic.claude-v2", + * region: "us-east-1", + * credentials: { + * accessKeyId: process.env.BEDROCK_AWS_ACCESS_KEY_ID!, + * secretAccessKey: process.env.BEDROCK_AWS_SECRET_ACCESS_KEY!, + * }, + * maxTokens: 150, + * temperature: 0.7, + * stopSequences: ["\n", " Human:", " Assistant:"], + * streaming: false, + * trace: "ENABLED", + * guardrailIdentifier: "your-guardrail-id", + * guardrailVersion: "1.0", + * guardrailConfig: { + * tagSuffix: "example", + * streamProcessingMode: "SYNCHRONOUS", + * }, + * }); + * + * // Prepare the message to be sent to the model + * const message = new HumanMessage("Tell me a joke"); + * + * // Invoke the model with the message + * const res = await model.invoke([message]); + * + * // Output the response from the model + * console.log(res); + * } + * + * run().catch(console.error); + * ``` + * + * For streaming responses, use the following example: + * @example + * ```typescript + * import { BedrockChat } from 'path-to-your-bedrock-chat-module'; + * import { HumanMessage } from '@langchain/core/messages'; + * + * async function runStreaming() { + * // Instantiate the BedrockChat model with the desired configuration + * const model = new BedrockChat({ + * model: "anthropic.claude-3-sonnet-20240229-v1:0", + * region: "us-east-1", + * credentials: { + * accessKeyId: process.env.BEDROCK_AWS_ACCESS_KEY_ID!, + * secretAccessKey: process.env.BEDROCK_AWS_SECRET_ACCESS_KEY!, + * }, + * maxTokens: 150, + * temperature: 0.7, + * stopSequences: ["\n", " Human:", " Assistant:"], + * streaming: true, + * trace: "ENABLED", + * guardrailIdentifier: "your-guardrail-id", + * guardrailVersion: "1.0", + * guardrailConfig: { + * tagSuffix: "example", + * streamProcessingMode: "SYNCHRONOUS", + * }, + * }); + * + * // Prepare the message to be sent to the model + * const message = new HumanMessage("Tell me a joke"); + * + * // Stream the response from the model + * const stream = await model.stream([message]); + * for await (const chunk of stream) { + * // Output each chunk of the response + * console.log(chunk); + * } + * } + * + * runStreaming().catch(console.error); * ``` */ export class BedrockChat extends BaseChatModel implements BaseBedrockInput { @@ -135,6 +214,17 @@ export class BedrockChat extends BaseChatModel implements BaseBedrockInput { lc_serializable = true; + trace?: "ENABLED" | "DISABLED"; + + guardrailIdentifier = ""; + + guardrailVersion = ""; + + guardrailConfig?: { + tagSuffix: string; + streamProcessingMode: "SYNCHRONOUS" | "ASYNCHRONOUS"; + }; + get lc_aliases(): Record { return { model: "model_id", @@ -209,11 +299,28 @@ export class BedrockChat extends BaseChatModel implements BaseBedrockInput { this.modelKwargs = fields?.modelKwargs; this.streaming = fields?.streaming ?? this.streaming; this.usesMessagesApi = canUseMessagesApi(this.model); + this.trace = fields?.trace ?? this.trace; + this.guardrailVersion = fields?.guardrailVersion ?? this.guardrailVersion; + this.guardrailIdentifier = + fields?.guardrailIdentifier ?? this.guardrailIdentifier; + this.guardrailConfig = fields?.guardrailConfig; + } + + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { + const params = this.invocationParams(options); + return { + ls_provider: "bedrock", + ls_model_name: this.model, + ls_model_type: "chat", + ls_temperature: params.temperature ?? undefined, + ls_max_tokens: params.max_tokens ?? undefined, + ls_stop: options.stop, + }; } async _generate( messages: BaseMessage[], - options: this["ParsedCallOptions"], + options: Partial, runManager?: CallbackManagerForLLMRun ): Promise { const service = "bedrock-runtime"; @@ -285,7 +392,8 @@ export class BedrockChat extends BaseChatModel implements BaseBedrockInput { this.maxTokens, this.temperature, options.stop ?? this.stopSequences, - this.modelKwargs + this.modelKwargs, + this.guardrailConfig ) : BedrockLLMInputOutputAdapter.prepareInput( provider, @@ -294,7 +402,8 @@ export class BedrockChat extends BaseChatModel implements BaseBedrockInput { this.temperature, options.stop ?? this.stopSequences, this.modelKwargs, - fields.bedrockMethod + fields.bedrockMethod, + this.guardrailConfig ); const url = new URL( @@ -313,6 +422,13 @@ export class BedrockChat extends BaseChatModel implements BaseBedrockInput { host: url.host, accept: "application/json", "content-type": "application/json", + ...(this.trace && + this.guardrailIdentifier && + this.guardrailVersion && { + "X-Amzn-Bedrock-Trace": this.trace, + "X-Amzn-Bedrock-GuardrailIdentifier": this.guardrailIdentifier, + "X-Amzn-Bedrock-GuardrailVersion": this.guardrailVersion, + }), }, }); diff --git a/libs/langchain-community/src/chat_models/fireworks.ts b/libs/langchain-community/src/chat_models/fireworks.ts index 65d6a6588c31..2fb7a62bd118 100644 --- a/libs/langchain-community/src/chat_models/fireworks.ts +++ b/libs/langchain-community/src/chat_models/fireworks.ts @@ -104,7 +104,7 @@ export class ChatFireworks extends ChatOpenAI { this.apiKey = fireworksApiKey; } - protected getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { const params = super.getLsParams(options); params.ls_provider = "fireworks"; return params; diff --git a/libs/langchain-community/src/chat_models/ollama.ts b/libs/langchain-community/src/chat_models/ollama.ts index 3fa78fa9c6b2..7c037864498a 100644 --- a/libs/langchain-community/src/chat_models/ollama.ts +++ b/libs/langchain-community/src/chat_models/ollama.ts @@ -177,7 +177,7 @@ export class ChatOllama this.format = fields.format; } - protected getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { const params = this.invocationParams(options); return { ls_provider: "ollama", diff --git a/libs/langchain-community/src/chat_models/tests/chatbedrock.int.test.ts b/libs/langchain-community/src/chat_models/tests/chatbedrock.int.test.ts index fa16be634da3..2d92d70f89c9 100644 --- a/libs/langchain-community/src/chat_models/tests/chatbedrock.int.test.ts +++ b/libs/langchain-community/src/chat_models/tests/chatbedrock.int.test.ts @@ -1,27 +1,14 @@ +// libs/langchain-community/src/chat_models/tests/chatbedrock.int.test.ts /* eslint-disable no-process-env */ /* eslint-disable @typescript-eslint/no-non-null-assertion */ import { test, expect } from "@jest/globals"; import { HumanMessage } from "@langchain/core/messages"; import { BedrockChat as BedrockChatWeb } from "../bedrock/web.js"; -import { BedrockChat } from "../bedrock/index.js"; - -// void testChatModel( -// "Test Bedrock chat model: Llama2 13B v1", -// "us-east-1", -// "meta.llama2-13b-chat-v1", -// "What is your name?" -// ); -// void testChatStreamingModel( -// "Test Bedrock streaming chat model: Llama2 13B v1", -// "us-east-1", -// "meta.llama2-13b-chat-v1", -// "What is your name and something about yourself?" -// ); void testChatModel( "Test Bedrock chat model Generating search queries: Command-r", - "us-east-1", + "us-west-2", "cohere.command-r-v1:0", "Who is more popular: Nsync or Backstreet Boys?", { @@ -31,14 +18,15 @@ void testChatModel( void testChatModel( "Test Bedrock chat model: Command-r", - "us-east-1", + "us-west-2", "cohere.command-r-v1:0", - "What is your name?" + "What is your name?", + {} ); void testChatModel( "Test Bedrock chat model: Command-r", - "us-east-1", + "us-west-2", "cohere.command-r-v1:0", "What are the characteristics of the emperor penguin?", { @@ -54,14 +42,15 @@ void testChatModel( void testChatStreamingModel( "Test Bedrock chat model streaming: Command-r", - "us-east-1", + "us-west-2", "cohere.command-r-v1:0", - "What is your name and something about yourself?" + "What is your name and something about yourself?", + {} ); void testChatStreamingModel( "Test Bedrock chat model streaming: Command-r", - "us-east-1", + "us-west-2", "cohere.command-r-v1:0", "What are the characteristics of the emperor penguin?", { @@ -77,73 +66,94 @@ void testChatStreamingModel( void testChatHandleLLMNewToken( "Test Bedrock chat model HandleLLMNewToken: Command-r", - "us-east-1", + "us-west-2", "cohere.command-r-v1:0", "What is your name and something about yourself?" ); void testChatModel( "Test Bedrock chat model: Mistral-7b-instruct", - "us-east-1", + "us-west-2", "mistral.mistral-7b-instruct-v0:2", - "What is your name?" + "What is your name?", + {} ); void testChatStreamingModel( "Test Bedrock chat model streaming: Mistral-7b-instruct", - "us-east-1", + "us-west-2", "mistral.mistral-7b-instruct-v0:2", - "What is your name and something about yourself?" + "What is your name and something about yourself?", + {} ); void testChatHandleLLMNewToken( "Test Bedrock chat model HandleLLMNewToken: Mistral-7b-instruct", - "us-east-1", + "us-west-2", "mistral.mistral-7b-instruct-v0:2", "What is your name and something about yourself?" ); void testChatModel( "Test Bedrock chat model: Claude-3", - "us-east-1", + "us-west-2", "anthropic.claude-3-sonnet-20240229-v1:0", - "What is your name?" + "What is your name?", + {} + // "ENABLED", + // "", + // "DRAFT", + // { tagSuffix: "test", streamProcessingMode: "SYNCHRONOUS" } ); void testChatStreamingModel( "Test Bedrock chat model streaming: Claude-3", - "us-east-1", + "us-west-2", "anthropic.claude-3-sonnet-20240229-v1:0", - "What is your name and something about yourself?" + "What is your name and something about yourself?", + {} + // "ENABLED", + // "", + // "DRAFT", + // { tagSuffix: "test", streamProcessingMode: "SYNCHRONOUS" } ); void testChatHandleLLMNewToken( "Test Bedrock chat model HandleLLMNewToken: Claude-3", - "us-east-1", + "us-west-2", "anthropic.claude-3-sonnet-20240229-v1:0", "What is your name and something about yourself?" + // "ENABLED", + // "", + // "DRAFT", + // { tagSuffix: "test", streamProcessingMode: "SYNCHRONOUS" } ); -// void testChatHandleLLMNewToken( -// "Test Bedrock chat model HandleLLMNewToken: Llama2 13B v1", -// "us-east-1", -// "meta.llama2-13b-chat-v1", -// "What is your name and something about yourself?" -// ); - /** * Tests a BedrockChat model * @param title The name of the test to run * @param defaultRegion The AWS region to default back to if not set via environment * @param model The model string to test * @param message The prompt test to send to the LLM + * @param modelKwargs Optional guardrail configuration + * @param trace Optional trace setting + * @param guardrailIdentifier Optional guardrail identifier + * @param guardrailVersion Optional guardrail version + * @param guardrailConfig Optional guardrail configuration */ async function testChatModel( title: string, defaultRegion: string, model: string, message: string, - modelKwargs?: Record + modelKwargs?: Record, + trace?: "ENABLED" | "DISABLED", + guardrailIdentifier?: string, + guardrailVersion?: string, + guardrailConfig?: { + tagSuffix: string; + streamProcessingMode: "SYNCHRONOUS" | "ASYNCHRONOUS"; + } ) { test(title, async () => { const region = process.env.BEDROCK_AWS_REGION ?? defaultRegion; @@ -156,28 +166,57 @@ async function testChatModel( credentials: { secretAccessKey: process.env.BEDROCK_AWS_SECRET_ACCESS_KEY!, accessKeyId: process.env.BEDROCK_AWS_ACCESS_KEY_ID!, - sessionToken: process.env.BEDROCK_AWS_SESSION_TOKEN, + // sessionToken: process.env.BEDROCK_AWS_SESSION_TOKEN, }, modelKwargs, + ...(trace && + guardrailIdentifier && + guardrailVersion && { + trace, + guardrailIdentifier, + guardrailVersion, + guardrailConfig, + }), }); const res = await bedrock.invoke([new HumanMessage(message)]); console.log(res); + + expect(res).toBeDefined(); + if (trace && guardrailIdentifier && guardrailVersion) { + expect(bedrock.trace).toBe(trace); + expect(bedrock.guardrailIdentifier).toBe(guardrailIdentifier); + expect(bedrock.guardrailVersion).toBe(guardrailVersion); + expect(bedrock.guardrailConfig).toEqual(guardrailConfig); + } }); } + /** * Tests a BedrockChat model with a streaming response * @param title The name of the test to run * @param defaultRegion The AWS region to default back to if not set via environment * @param model The model string to test * @param message The prompt test to send to the LLM + * @param modelKwargs Optional guardrail configuration + * @param trace Optional trace setting + * @param guardrailIdentifier Optional guardrail identifier + * @param guardrailVersion Optional guardrail version + * @param guardrailConfig Optional guardrail configuration */ async function testChatStreamingModel( title: string, defaultRegion: string, model: string, message: string, - modelKwargs?: Record + modelKwargs?: Record, + trace?: "ENABLED" | "DISABLED", + guardrailIdentifier?: string, + guardrailVersion?: string, + guardrailConfig?: { + tagSuffix: string; + streamProcessingMode: "SYNCHRONOUS" | "ASYNCHRONOUS"; + } ) { test(title, async () => { const region = process.env.BEDROCK_AWS_REGION ?? defaultRegion; @@ -190,9 +229,17 @@ async function testChatStreamingModel( credentials: { secretAccessKey: process.env.BEDROCK_AWS_SECRET_ACCESS_KEY!, accessKeyId: process.env.BEDROCK_AWS_ACCESS_KEY_ID!, - sessionToken: process.env.BEDROCK_AWS_SESSION_TOKEN, + // sessionToken: process.env.BEDROCK_AWS_SESSION_TOKEN, }, modelKwargs, + ...(trace && + guardrailIdentifier && + guardrailVersion && { + trace, + guardrailIdentifier, + guardrailVersion, + guardrailConfig, + }), }); const stream = await bedrock.stream([ @@ -208,18 +255,30 @@ async function testChatStreamingModel( expect(chunks.length).toBeGreaterThan(1); }); } + /** * Tests a BedrockChat model with a streaming response using a new token callback * @param title The name of the test to run * @param defaultRegion The AWS region to default back to if not set via environment * @param model The model string to test * @param message The prompt test to send to the LLM + * @param trace Optional trace setting + * @param guardrailIdentifier Optional guardrail identifier + * @param guardrailVersion Optional guardrail version + * @param guardrailConfig Optional guardrail configuration */ async function testChatHandleLLMNewToken( title: string, defaultRegion: string, model: string, - message: string + message: string, + trace?: "ENABLED" | "DISABLED", + guardrailIdentifier?: string, + guardrailVersion?: string, + guardrailConfig?: { + tagSuffix: string; + streamProcessingMode: "SYNCHRONOUS" | "ASYNCHRONOUS"; + } ) { test(title, async () => { const region = process.env.BEDROCK_AWS_REGION ?? defaultRegion; @@ -233,7 +292,7 @@ async function testChatHandleLLMNewToken( credentials: { secretAccessKey: process.env.BEDROCK_AWS_SECRET_ACCESS_KEY!, accessKeyId: process.env.BEDROCK_AWS_ACCESS_KEY_ID!, - sessionToken: process.env.BEDROCK_AWS_SESSION_TOKEN, + // sessionToken: process.env.BEDROCK_AWS_SESSION_TOKEN, }, streaming: true, callbacks: [ @@ -246,6 +305,14 @@ async function testChatHandleLLMNewToken( }, }, ], + ...(trace && + guardrailIdentifier && + guardrailVersion && { + trace, + guardrailIdentifier, + guardrailVersion, + guardrailConfig, + }), }); const stream = await bedrock.invoke([new HumanMessage(message)]); expect(tokens.length).toBeGreaterThan(1); @@ -259,7 +326,7 @@ test.skip.each([ // "amazon.titan-text-lite-v1", // "amazon.titan-text-agile-v1", ])("Test Bedrock base chat model: %s", async (model) => { - const region = process.env.BEDROCK_AWS_REGION ?? "us-east-1"; + const region = process.env.BEDROCK_AWS_REGION ?? "us-west-2"; const bedrock = new BedrockChatWeb({ region, @@ -269,7 +336,7 @@ test.skip.each([ credentials: { secretAccessKey: process.env.BEDROCK_AWS_SECRET_ACCESS_KEY!, accessKeyId: process.env.BEDROCK_AWS_ACCESS_KEY_ID!, - sessionToken: process.env.BEDROCK_AWS_SESSION_TOKEN, + // sessionToken: process.env.BEDROCK_AWS_SESSION_TOKEN, }, }); @@ -278,14 +345,3 @@ test.skip.each([ expect(res.content.length).toBeGreaterThan(1); }); - -test.skip("new credential fields", async () => { - const model = new BedrockChat({ - filepath: - "/Users/bracesproul/code/lang-chain-ai/langchainjs/libs/langchain-community/src/chat_models/tests/aws_credentials", - model: "anthropic.claude-3-sonnet-20240229-v1:0", - region: process.env.BEDROCK_AWS_REGION, - }); - const res = await model.invoke(["Why is the sky blue? Be VERY concise!"]); - console.log("res", res); -}); diff --git a/libs/langchain-community/src/chat_models/tests/chatbedrock.standard.int.test.ts b/libs/langchain-community/src/chat_models/tests/chatbedrock.standard.int.test.ts new file mode 100644 index 000000000000..f1e1808632b9 --- /dev/null +++ b/libs/langchain-community/src/chat_models/tests/chatbedrock.standard.int.test.ts @@ -0,0 +1,51 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelIntegrationTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { BaseChatModelCallOptions } from "@langchain/core/language_models/chat_models"; +import { BedrockChat } from "../bedrock/index.js"; + +class BedrockChatStandardIntegrationTests extends ChatModelIntegrationTests< + BaseChatModelCallOptions, + AIMessageChunk +> { + constructor() { + const region = process.env.BEDROCK_AWS_REGION ?? "us-east-1"; + super({ + Cls: BedrockChat, + chatModelHasToolCalling: false, + chatModelHasStructuredOutput: false, + constructorArgs: { + region, + model: "amazon.titan-text-express-v1", + credentials: { + secretAccessKey: process.env.BEDROCK_AWS_SECRET_ACCESS_KEY, + accessKeyId: process.env.BEDROCK_AWS_ACCESS_KEY_ID, + }, + }, + }); + } + + async testUsageMetadataStreaming() { + this.skipTestMessage( + "testUsageMetadataStreaming", + "BedrockChat", + "Streaming tokens is not currently supported." + ); + } + + async testUsageMetadata() { + this.skipTestMessage( + "testUsageMetadata", + "BedrockChat", + "Usage metadata tokens is not currently supported." + ); + } +} + +const testClass = new BedrockChatStandardIntegrationTests(); + +test("BedrockChatStandardIntegrationTests", async () => { + const testResults = await testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-community/src/chat_models/tests/chatbedrock.standard.test.ts b/libs/langchain-community/src/chat_models/tests/chatbedrock.standard.test.ts new file mode 100644 index 000000000000..f2ce23db39ba --- /dev/null +++ b/libs/langchain-community/src/chat_models/tests/chatbedrock.standard.test.ts @@ -0,0 +1,39 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelUnitTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { BaseChatModelCallOptions } from "@langchain/core/language_models/chat_models"; +import { BedrockChat } from "../bedrock/index.js"; + +class BedrockChatStandardUnitTests extends ChatModelUnitTests< + BaseChatModelCallOptions, + AIMessageChunk +> { + constructor() { + super({ + Cls: BedrockChat, + chatModelHasToolCalling: false, + chatModelHasStructuredOutput: false, + constructorArgs: {}, + }); + process.env.BEDROCK_AWS_SECRET_ACCESS_KEY = "test"; + process.env.BEDROCK_AWS_ACCESS_KEY_ID = "test"; + process.env.BEDROCK_AWS_SESSION_TOKEN = "test"; + process.env.AWS_DEFAULT_REGION = "us-east-1"; + } + + testChatModelInitApiKey() { + this.skipTestMessage( + "testChatModelInitApiKey", + "BedrockChat", + this.multipleApiKeysRequiredMessage + ); + } +} + +const testClass = new BedrockChatStandardUnitTests(); + +test("BedrockChatStandardUnitTests", () => { + const testResults = testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-community/src/chat_models/tests/chatbedrock.test.ts b/libs/langchain-community/src/chat_models/tests/chatbedrock.test.ts index df5d7c50ea67..4caba4ee3a5c 100644 --- a/libs/langchain-community/src/chat_models/tests/chatbedrock.test.ts +++ b/libs/langchain-community/src/chat_models/tests/chatbedrock.test.ts @@ -4,14 +4,17 @@ import { BedrockChat } from "../bedrock/web.js"; test("Test Bedrock identifying params", async () => { - const region = "us-east-1"; - const model = "anthropic.claude-v2"; + const region = "us-west-2"; + const model = "anthropic.claude-3-sonnet-20240229-v1:0"; const bedrock = new BedrockChat({ maxTokens: 20, region, model, maxRetries: 0, + trace: "ENABLED", + guardrailIdentifier: "define", + guardrailVersion: "DRAFT", credentials: { accessKeyId: "unused", secretAccessKey: "unused", diff --git a/libs/langchain-community/src/chat_models/tests/chatfireworks.standard.int.test.ts b/libs/langchain-community/src/chat_models/tests/chatfireworks.standard.int.test.ts new file mode 100644 index 000000000000..be96b975b7c2 --- /dev/null +++ b/libs/langchain-community/src/chat_models/tests/chatfireworks.standard.int.test.ts @@ -0,0 +1,36 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelIntegrationTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { ChatFireworks, ChatFireworksCallOptions } from "../fireworks.js"; + +class ChatFireworksStandardIntegrationTests extends ChatModelIntegrationTests< + ChatFireworksCallOptions, + AIMessageChunk +> { + constructor() { + super({ + Cls: ChatFireworks, + chatModelHasToolCalling: true, + chatModelHasStructuredOutput: true, + constructorArgs: { + model: "accounts/fireworks/models/firefunction-v1", + }, + }); + } + + async testToolMessageHistoriesListContent() { + this.skipTestMessage( + "testToolMessageHistoriesListContent", + "ChatFireworks", + "Not implemented." + ); + } +} + +const testClass = new ChatFireworksStandardIntegrationTests(); + +test("ChatFireworksStandardIntegrationTests", async () => { + const testResults = await testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-community/src/chat_models/tests/chatfireworks.standard.test.ts b/libs/langchain-community/src/chat_models/tests/chatfireworks.standard.test.ts new file mode 100644 index 000000000000..bde920910676 --- /dev/null +++ b/libs/langchain-community/src/chat_models/tests/chatfireworks.standard.test.ts @@ -0,0 +1,36 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelUnitTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { ChatFireworks, ChatFireworksCallOptions } from "../fireworks.js"; + +class ChatFireworksStandardUnitTests extends ChatModelUnitTests< + ChatFireworksCallOptions, + AIMessageChunk +> { + constructor() { + super({ + Cls: ChatFireworks, + chatModelHasToolCalling: true, + chatModelHasStructuredOutput: true, + constructorArgs: {}, + }); + process.env.FIREWORKS_API_KEY = "test"; + } + + testChatModelInitApiKey() { + // Unset the API key env var here so this test can properly check + // the API key class arg. + process.env.FIREWORKS_API_KEY = ""; + super.testChatModelInitApiKey(); + // Re-set the API key env var here so other tests can run properly. + process.env.FIREWORKS_API_KEY = "test"; + } +} + +const testClass = new ChatFireworksStandardUnitTests(); + +test("ChatFireworksStandardUnitTests", () => { + const testResults = testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-community/src/chat_models/tests/chatollama.standard.test.ts b/libs/langchain-community/src/chat_models/tests/chatollama.standard.test.ts new file mode 100644 index 000000000000..69f119fbc7d0 --- /dev/null +++ b/libs/langchain-community/src/chat_models/tests/chatollama.standard.test.ts @@ -0,0 +1,34 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelUnitTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { ChatOllama, ChatOllamaCallOptions } from "../ollama.js"; + +class ChatOllamaStandardUnitTests extends ChatModelUnitTests< + ChatOllamaCallOptions, + AIMessageChunk +> { + constructor() { + super({ + Cls: ChatOllama, + chatModelHasToolCalling: false, + chatModelHasStructuredOutput: false, + constructorArgs: {}, + }); + } + + testChatModelInitApiKey() { + this.skipTestMessage( + "testChatModelInitApiKey", + "ChatOllama", + "API key not required." + ); + } +} + +const testClass = new ChatOllamaStandardUnitTests(); + +test("ChatOllamaStandardUnitTests", () => { + const testResults = testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-community/src/chat_models/tests/chattogetherai.standard.int.test.ts b/libs/langchain-community/src/chat_models/tests/chattogetherai.standard.int.test.ts new file mode 100644 index 000000000000..885bc169c6ec --- /dev/null +++ b/libs/langchain-community/src/chat_models/tests/chattogetherai.standard.int.test.ts @@ -0,0 +1,26 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelIntegrationTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { ChatTogetherAI, ChatTogetherAICallOptions } from "../togetherai.js"; + +class ChatTogetherAIStandardIntegrationTests extends ChatModelIntegrationTests< + ChatTogetherAICallOptions, + AIMessageChunk +> { + constructor() { + super({ + Cls: ChatTogetherAI, + chatModelHasToolCalling: true, + chatModelHasStructuredOutput: true, + constructorArgs: {}, + }); + } +} + +const testClass = new ChatTogetherAIStandardIntegrationTests(); + +test("ChatTogetherAIStandardIntegrationTests", async () => { + const testResults = await testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-community/src/chat_models/tests/chattogetherai.standard.test.ts b/libs/langchain-community/src/chat_models/tests/chattogetherai.standard.test.ts new file mode 100644 index 000000000000..2fc8d1bfb71e --- /dev/null +++ b/libs/langchain-community/src/chat_models/tests/chattogetherai.standard.test.ts @@ -0,0 +1,36 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelUnitTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { ChatTogetherAI, ChatTogetherAICallOptions } from "../togetherai.js"; + +class ChatTogetherAIStandardUnitTests extends ChatModelUnitTests< + ChatTogetherAICallOptions, + AIMessageChunk +> { + constructor() { + super({ + Cls: ChatTogetherAI, + chatModelHasToolCalling: false, + chatModelHasStructuredOutput: false, + constructorArgs: {}, + }); + process.env.TOGETHER_AI_API_KEY = "test"; + } + + testChatModelInitApiKey() { + // Unset the API key env var here so this test can properly check + // the API key class arg. + process.env.TOGETHER_AI_API_KEY = ""; + super.testChatModelInitApiKey(); + // Re-set the API key env var here so other tests can run properly. + process.env.TOGETHER_AI_API_KEY = "test"; + } +} + +const testClass = new ChatTogetherAIStandardUnitTests(); + +test("ChatTogetherAIStandardUnitTests", () => { + const testResults = testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-community/src/chat_models/togetherai.ts b/libs/langchain-community/src/chat_models/togetherai.ts index 5488e6cf61fe..b377024406a1 100644 --- a/libs/langchain-community/src/chat_models/togetherai.ts +++ b/libs/langchain-community/src/chat_models/togetherai.ts @@ -116,7 +116,7 @@ export class ChatTogetherAI extends ChatOpenAI { }); } - protected getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { const params = super.getLsParams(options); params.ls_provider = "together"; return params; diff --git a/libs/langchain-community/src/chat_models/webllm.ts b/libs/langchain-community/src/chat_models/webllm.ts index e56d10f1f40f..e3ecda6bd1c1 100644 --- a/libs/langchain-community/src/chat_models/webllm.ts +++ b/libs/langchain-community/src/chat_models/webllm.ts @@ -43,7 +43,7 @@ export interface WebLLMCallOptions extends BaseLanguageModelCallOptions {} export class ChatWebLLM extends SimpleChatModel { static inputs: WebLLMInputs; - protected engine: webllm.EngineInterface; + protected engine: webllm.MLCEngine; appConfig?: webllm.AppConfig; @@ -63,6 +63,7 @@ export class ChatWebLLM extends SimpleChatModel { this.chatOptions = inputs.chatOptions; this.model = inputs.model; this.temperature = inputs.temperature; + this.engine = new webllm.MLCEngine(); } _llmType() { @@ -70,12 +71,10 @@ export class ChatWebLLM extends SimpleChatModel { } async initialize(progressCallback?: webllm.InitProgressCallback) { - this.engine = new webllm.Engine(); if (progressCallback !== undefined) { this.engine.setInitProgressCallback(progressCallback); } await this.reload(this.model, this.chatOptions, this.appConfig); - this.engine.setInitProgressCallback(() => {}); } async reload( @@ -83,11 +82,7 @@ export class ChatWebLLM extends SimpleChatModel { newAppConfig?: webllm.AppConfig, newChatOpts?: webllm.ChatOptions ) { - if (this.engine !== undefined) { - await this.engine.reload(modelId, newAppConfig, newChatOpts); - } else { - throw new Error("Initialize model before reloading."); - } + await this.engine.reload(modelId, newChatOpts, newAppConfig); } async *_streamResponseChunks( @@ -95,8 +90,6 @@ export class ChatWebLLM extends SimpleChatModel { options: this["ParsedCallOptions"], runManager?: CallbackManagerForLLMRun ): AsyncGenerator { - await this.initialize(); - const messagesInput: ChatCompletionMessageParam[] = messages.map( (message) => { if (typeof message.content !== "string") { @@ -124,15 +117,12 @@ export class ChatWebLLM extends SimpleChatModel { } ); - const stream = this.engine.chatCompletionAsyncChunkGenerator( - { - stream: true, - messages: messagesInput, - stop: options.stop, - logprobs: true, - }, - {} - ); + const stream = await this.engine.chat.completions.create({ + stream: true, + messages: messagesInput, + stop: options.stop, + logprobs: true, + }); for await (const chunk of stream) { // Last chunk has undefined content const text = chunk.choices[0].delta.content ?? ""; @@ -146,7 +136,7 @@ export class ChatWebLLM extends SimpleChatModel { }, }), }); - await runManager?.handleLLMNewToken(text ?? ""); + await runManager?.handleLLMNewToken(text); } } diff --git a/libs/langchain-community/src/utils/bedrock.ts b/libs/langchain-community/src/utils/bedrock.ts index e779eeacc989..b9243679c2f5 100644 --- a/libs/langchain-community/src/utils/bedrock.ts +++ b/libs/langchain-community/src/utils/bedrock.ts @@ -222,6 +222,21 @@ export interface BaseBedrockInput { /** Whether or not to stream responses */ streaming: boolean; + + /** Trace settings for the Bedrock Guardrails. */ + trace?: "ENABLED" | "DISABLED"; + + /** Identifier for the guardrail configuration. */ + guardrailIdentifier?: string; + + /** Version for the guardrail configuration. */ + guardrailVersion?: string; + + /** Required when Guardrail is in use. */ + guardrailConfig?: { + tagSuffix: string; + streamProcessingMode: "SYNCHRONOUS" | "ASYNCHRONOUS"; + }; } type Dict = { [key: string]: unknown }; @@ -244,7 +259,13 @@ export class BedrockLLMInputOutputAdapter { temperature = 0, stopSequences: string[] | undefined = undefined, modelKwargs: Record = {}, - bedrockMethod: "invoke" | "invoke-with-response-stream" = "invoke" + bedrockMethod: "invoke" | "invoke-with-response-stream" = "invoke", + guardrailConfig: + | { + tagSuffix: string; + streamProcessingMode: "SYNCHRONOUS" | "ASYNCHRONOUS"; + } + | undefined = undefined ): Dict { const inputBody: Dict = {}; @@ -282,6 +303,15 @@ export class BedrockLLMInputOutputAdapter { inputBody.temperature = temperature; inputBody.stop = stopSequences; } + + if ( + guardrailConfig && + guardrailConfig.tagSuffix && + guardrailConfig.streamProcessingMode + ) { + inputBody["amazon-bedrock-guardrailConfig"] = guardrailConfig; + } + return { ...inputBody, ...modelKwargs }; } @@ -291,7 +321,13 @@ export class BedrockLLMInputOutputAdapter { maxTokens = 1024, temperature = 0, stopSequences: string[] | undefined = undefined, - modelKwargs: Record = {} + modelKwargs: Record = {}, + guardrailConfig: + | { + tagSuffix: string; + streamProcessingMode: "SYNCHRONOUS" | "ASYNCHRONOUS"; + } + | undefined = undefined ): Dict { const inputBody: Dict = {}; @@ -306,7 +342,6 @@ export class BedrockLLMInputOutputAdapter { inputBody.max_tokens = maxTokens; inputBody.temperature = temperature; inputBody.stop_sequences = stopSequences; - return { ...inputBody, ...modelKwargs }; } else if (provider === "cohere") { const { system, @@ -322,12 +357,21 @@ export class BedrockLLMInputOutputAdapter { inputBody.max_tokens = maxTokens; inputBody.temperature = temperature; inputBody.stop_sequences = stopSequences; - return { ...inputBody, ...modelKwargs }; } else { throw new Error( "The messages API is currently only supported by Anthropic or Cohere" ); } + + if ( + guardrailConfig && + guardrailConfig.tagSuffix && + guardrailConfig.streamProcessingMode + ) { + inputBody["amazon-bedrock-guardrailConfig"] = guardrailConfig; + } + + return { ...inputBody, ...modelKwargs }; } /** @@ -484,6 +528,7 @@ function parseMessage(responseBody: any, asChunk?: boolean): ChatGeneration { } function parseMessageCohere( + // eslint-disable-next-line @typescript-eslint/no-explicit-any responseBody: any, asChunk?: boolean ): ChatGeneration { diff --git a/libs/langchain-google-common/src/chat_models.ts b/libs/langchain-google-common/src/chat_models.ts index 1a27b3cd2cab..4f65b13a46b8 100644 --- a/libs/langchain-google-common/src/chat_models.ts +++ b/libs/langchain-google-common/src/chat_models.ts @@ -231,7 +231,7 @@ export abstract class ChatGoogleBase this.buildConnection(fields ?? {}, client); } - protected getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { const params = this.invocationParams(options); return { ls_provider: "google_vertexai", diff --git a/libs/langchain-google-gauth/package.json b/libs/langchain-google-gauth/package.json index 57b9db09a991..341d5c22d750 100644 --- a/libs/langchain-google-gauth/package.json +++ b/libs/langchain-google-gauth/package.json @@ -16,26 +16,21 @@ "scripts": { "build": "yarn turbo:command build:internal --filter=@langchain/google-gauth", "build:internal": "yarn lc-build:v2 --create-entrypoints --pre --tree-shaking", - "build:deps": "yarn run turbo:command build --filter=@langchain/google-common", - "build:esm": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist/ && rm -rf dist/tests dist/**/tests", - "build:cjs": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist-cjs/ -p tsconfig.cjs.json && yarn move-cjs-to-dist && rm -rf dist-cjs", - "build:watch": "yarn create-entrypoints && tsc --outDir dist/ --watch", - "build:scripts": "yarn create-entrypoints && yarn check-tree-shaking", "lint:eslint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/", "lint:dpdm": "dpdm --exit-code circular:1 --no-warning --no-tree src/*.ts src/**/*.ts", "lint": "yarn lint:eslint && yarn lint:dpdm", "lint:fix": "yarn lint:eslint --fix && yarn lint:dpdm", "clean": "rm -rf .turbo dist/", "prepack": "yarn build", - "test": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%", - "test:watch": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", - "test:single": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", - "test:integration": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test": "NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%", + "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", + "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", + "test:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:unit": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard": "yarn test:standard:unit && yarn test:standard:int", "format": "prettier --config .prettierrc --write \"src\"", - "format:check": "prettier --config .prettierrc --check \"src\"", - "move-cjs-to-dist": "yarn lc-build --config ./langchain.config.js --move-cjs-dist", - "create-entrypoints": "yarn lc-build --config ./langchain.config.js --create-entrypoints", - "check-tree-shaking": "yarn lc-build --config ./langchain.config.js --tree-shaking" + "format:check": "prettier --config .prettierrc --check \"src\"" }, "author": "LangChain", "license": "MIT", @@ -47,6 +42,7 @@ "devDependencies": { "@jest/globals": "^29.5.0", "@langchain/scripts": "~0.0.14", + "@langchain/standard-tests": "workspace:*", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "@tsconfig/recommended": "^1.0.3", diff --git a/libs/langchain-google-genai/package.json b/libs/langchain-google-genai/package.json index d3443dbfbdfa..b5890cefd885 100644 --- a/libs/langchain-google-genai/package.json +++ b/libs/langchain-google-genai/package.json @@ -16,26 +16,21 @@ "scripts": { "build": "yarn turbo:command build:internal --filter=@langchain/google-genai", "build:internal": "yarn lc-build:v2 --create-entrypoints --pre --tree-shaking", - "build:deps": "yarn run turbo:command build --filter=@langchain/core", - "build:esm": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist/ && rm -rf dist/tests dist/**/tests", - "build:cjs": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist-cjs/ -p tsconfig.cjs.json && yarn move-cjs-to-dist && rm -rf dist-cjs", - "build:watch": "yarn create-entrypoints && tsc --outDir dist/ --watch", - "build:scripts": "yarn create-entrypoints && yarn check-tree-shaking", "lint:eslint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/", "lint:dpdm": "dpdm --exit-code circular:1 --no-warning --no-tree src/*.ts src/**/*.ts", "lint": "yarn lint:eslint && yarn lint:dpdm", "lint:fix": "yarn lint:eslint --fix && yarn lint:dpdm", "clean": "rm -rf .turbo dist/", "prepack": "yarn build", - "test": "yarn build:deps && NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%", - "test:watch": "yarn build:deps && NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", - "test:single": "yarn build:deps && NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", - "test:int": "yarn build:deps && NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test": "NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%", + "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", + "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", + "test:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:unit": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard": "yarn test:standard:unit && yarn test:standard:int", "format": "prettier --config .prettierrc --write \"src\"", - "format:check": "prettier --config .prettierrc --check \"src\"", - "move-cjs-to-dist": "yarn lc-build --config ./langchain.config.js --move-cjs-dist", - "create-entrypoints": "yarn lc-build --config ./langchain.config.js --create-entrypoints", - "check-tree-shaking": "yarn lc-build --config ./langchain.config.js --tree-shaking" + "format:check": "prettier --config .prettierrc --check \"src\"" }, "author": "LangChain", "license": "MIT", @@ -47,6 +42,7 @@ "devDependencies": { "@jest/globals": "^29.5.0", "@langchain/scripts": "~0.0.14", + "@langchain/standard-tests": "workspace:*", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "@tsconfig/recommended": "^1.0.3", diff --git a/libs/langchain-google-genai/src/chat_models.ts b/libs/langchain-google-genai/src/chat_models.ts index 22b76ad5d4e2..73b54d22bfab 100644 --- a/libs/langchain-google-genai/src/chat_models.ts +++ b/libs/langchain-google-genai/src/chat_models.ts @@ -308,7 +308,7 @@ export class ChatGoogleGenerativeAI ); } - protected getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { return { ls_provider: "google_genai", ls_model_name: this.model, diff --git a/libs/langchain-google-genai/src/tests/chat_models.standard.int.test.ts b/libs/langchain-google-genai/src/tests/chat_models.standard.int.test.ts new file mode 100644 index 000000000000..4f9909358165 --- /dev/null +++ b/libs/langchain-google-genai/src/tests/chat_models.standard.int.test.ts @@ -0,0 +1,76 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelIntegrationTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { + ChatGoogleGenerativeAI, + GoogleGenerativeAIChatCallOptions, +} from "../chat_models.js"; + +class ChatGoogleGenerativeAIStandardIntegrationTests extends ChatModelIntegrationTests< + GoogleGenerativeAIChatCallOptions, + AIMessageChunk +> { + constructor() { + if (!process.env.GOOGLE_API_KEY) { + throw new Error( + "Can not run Google Generative AI integration tests because GOOGLE_API_KEY is set" + ); + } + super({ + Cls: ChatGoogleGenerativeAI, + chatModelHasToolCalling: true, + chatModelHasStructuredOutput: true, + constructorArgs: { + maxRetries: 1, + }, + }); + } + + async testUsageMetadataStreaming() { + this.skipTestMessage( + "testUsageMetadataStreaming", + "ChatGoogleGenerativeAI", + "Streaming tokens is not currently supported." + ); + } + + async testUsageMetadata() { + this.skipTestMessage( + "testUsageMetadata", + "ChatGoogleGenerativeAI", + "Usage metadata tokens is not currently supported." + ); + } + + async testToolMessageHistoriesStringContent() { + this.skipTestMessage( + "testToolMessageHistoriesStringContent", + "ChatGoogleGenerativeAI", + "Not implemented." + ); + } + + async testToolMessageHistoriesListContent() { + this.skipTestMessage( + "testToolMessageHistoriesListContent", + "ChatGoogleGenerativeAI", + "Not implemented." + ); + } + + async testStructuredFewShotExamples() { + this.skipTestMessage( + "testStructuredFewShotExamples", + "ChatGoogleGenerativeAI", + ".bindTools not implemented properly." + ); + } +} + +const testClass = new ChatGoogleGenerativeAIStandardIntegrationTests(); + +test("ChatGoogleGenerativeAIStandardIntegrationTests", async () => { + const testResults = await testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-google-genai/src/tests/chat_models.standard.test.ts b/libs/langchain-google-genai/src/tests/chat_models.standard.test.ts new file mode 100644 index 000000000000..a7fa2ef2a3ae --- /dev/null +++ b/libs/langchain-google-genai/src/tests/chat_models.standard.test.ts @@ -0,0 +1,42 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelUnitTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { + ChatGoogleGenerativeAI, + GoogleGenerativeAIChatCallOptions, +} from "../chat_models.js"; + +class ChatGoogleGenerativeAIStandardUnitTests extends ChatModelUnitTests< + GoogleGenerativeAIChatCallOptions, + AIMessageChunk +> { + constructor() { + super({ + Cls: ChatGoogleGenerativeAI, + chatModelHasToolCalling: true, + chatModelHasStructuredOutput: true, + constructorArgs: {}, + }); + // This must be set so method like `.bindTools` or `.withStructuredOutput` + // which we call after instantiating the model will work. + // (constructor will throw if API key is not set) + process.env.GOOGLE_API_KEY = "test"; + } + + testChatModelInitApiKey() { + // Unset the API key env var here so this test can properly check + // the API key class arg. + process.env.GOOGLE_API_KEY = ""; + super.testChatModelInitApiKey(); + // Re-set the API key env var here so other tests can run properly. + process.env.GOOGLE_API_KEY = "test"; + } +} + +const testClass = new ChatGoogleGenerativeAIStandardUnitTests(); + +test("ChatGoogleGenerativeAIStandardUnitTests", () => { + const testResults = testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-groq/package.json b/libs/langchain-groq/package.json index 38f5ab175c44..8672bb54dc31 100644 --- a/libs/langchain-groq/package.json +++ b/libs/langchain-groq/package.json @@ -16,11 +16,6 @@ "scripts": { "build": "yarn turbo:command build:internal --filter=@langchain/groq", "build:internal": "yarn lc-build:v2 --create-entrypoints --pre --tree-shaking", - "build:deps": "yarn run turbo:command build --filter=@langchain/core", - "build:esm": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist/ && rm -rf dist/tests dist/**/tests", - "build:cjs": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist-cjs/ -p tsconfig.cjs.json && yarn move-cjs-to-dist && rm -rf dist-cjs", - "build:watch": "yarn create-entrypoints && tsc --outDir dist/ --watch", - "build:scripts": "yarn create-entrypoints && yarn check-tree-shaking", "lint:eslint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/", "lint:dpdm": "dpdm --exit-code circular:1 --no-warning --no-tree src/*.ts src/**/*.ts", "lint": "yarn lint:eslint && yarn lint:dpdm", @@ -31,11 +26,11 @@ "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", "test:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:unit": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard": "yarn test:standard:unit && yarn test:standard:int", "format": "prettier --config .prettierrc --write \"src\"", - "format:check": "prettier --config .prettierrc --check \"src\"", - "move-cjs-to-dist": "yarn lc-build --config ./langchain.config.js --move-cjs-dist", - "create-entrypoints": "yarn lc-build --config ./langchain.config.js --create-entrypoints", - "check-tree-shaking": "yarn lc-build --config ./langchain.config.js --tree-shaking" + "format:check": "prettier --config .prettierrc --check \"src\"" }, "author": "LangChain", "license": "MIT", @@ -50,6 +45,7 @@ "@jest/globals": "^29.5.0", "@langchain/openai": "workspace:^", "@langchain/scripts": "~0.0.14", + "@langchain/standard-tests": "workspace:*", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "@tsconfig/recommended": "^1.0.3", diff --git a/libs/langchain-groq/src/chat_models.ts b/libs/langchain-groq/src/chat_models.ts index 7dce52d8436b..acc43c2850d9 100644 --- a/libs/langchain-groq/src/chat_models.ts +++ b/libs/langchain-groq/src/chat_models.ts @@ -5,6 +5,7 @@ import { CallbackManagerForLLMRun } from "@langchain/core/callbacks/manager"; import { BaseChatModel, BaseChatModelCallOptions, + LangSmithParams, type BaseChatModelParams, } from "@langchain/core/language_models/chat_models"; import { @@ -312,6 +313,18 @@ export class ChatGroq extends BaseChatModel< this.maxTokens = fields?.maxTokens; } + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { + const params = this.invocationParams(options); + return { + ls_provider: "groq", + ls_model_name: this.model, + ls_model_type: "chat", + ls_temperature: params.temperature, + ls_max_tokens: params.max_tokens, + ls_stop: options.stop, + }; + } + async completionWithRetry( request: ChatCompletionCreateParamsStreaming, options?: OpenAICoreRequestOptions @@ -411,13 +424,24 @@ export class ChatGroq extends BaseChatModel< headers: options?.headers, } ); + let role = ""; for await (const data of response) { const choice = data?.choices[0]; if (!choice) { continue; } + // The `role` field is populated in the first delta of the response + // but is not present in subsequent deltas. Extract it when available. + if (choice.delta?.role) { + role = choice.delta.role; + } const chunk = new ChatGenerationChunk({ - message: _convertDeltaToMessageChunk(choice.delta ?? {}), + message: _convertDeltaToMessageChunk( + { + ...choice.delta, + role, + } ?? {} + ), text: choice.delta.content ?? "", generationInfo: { finishReason: choice.finish_reason, diff --git a/libs/langchain-groq/src/tests/chat_models.standard.int.test.ts b/libs/langchain-groq/src/tests/chat_models.standard.int.test.ts new file mode 100644 index 000000000000..4eb55e6cd5c7 --- /dev/null +++ b/libs/langchain-groq/src/tests/chat_models.standard.int.test.ts @@ -0,0 +1,57 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelIntegrationTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { ChatGroq, ChatGroqCallOptions } from "../chat_models.js"; + +class ChatGroqStandardIntegrationTests extends ChatModelIntegrationTests< + ChatGroqCallOptions, + AIMessageChunk +> { + constructor() { + if (!process.env.GROQ_API_KEY) { + throw new Error( + "Can not run Groq integration tests because GROQ_API_KEY is not set" + ); + } + super({ + Cls: ChatGroq, + chatModelHasToolCalling: true, + chatModelHasStructuredOutput: true, + constructorArgs: { + model: "mixtral-8x7b-32768", + }, + }); + } + + async testUsageMetadataStreaming() { + this.skipTestMessage( + "testUsageMetadataStreaming", + "ChatGroq", + "Streaming tokens is not currently supported." + ); + } + + async testUsageMetadata() { + this.skipTestMessage( + "testUsageMetadata", + "ChatGroq", + "Usage metadata tokens is not currently supported." + ); + } + + async testToolMessageHistoriesListContent() { + this.skipTestMessage( + "testToolMessageHistoriesListContent", + "ChatGroq", + "Not properly implemented." + ); + } +} + +const testClass = new ChatGroqStandardIntegrationTests(); + +test("ChatGroqStandardIntegrationTests", async () => { + const testResults = await testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-groq/src/tests/chat_models.standard.test.ts b/libs/langchain-groq/src/tests/chat_models.standard.test.ts new file mode 100644 index 000000000000..8678bfc87ca2 --- /dev/null +++ b/libs/langchain-groq/src/tests/chat_models.standard.test.ts @@ -0,0 +1,39 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelUnitTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { ChatGroq, ChatGroqCallOptions } from "../chat_models.js"; + +class ChatGroqStandardUnitTests extends ChatModelUnitTests< + ChatGroqCallOptions, + AIMessageChunk +> { + constructor() { + super({ + Cls: ChatGroq, + chatModelHasToolCalling: true, + chatModelHasStructuredOutput: true, + constructorArgs: {}, + }); + // This must be set so method like `.bindTools` or `.withStructuredOutput` + // which we call after instantiating the model will work. + // (constructor will throw if API key is not set) + process.env.GROQ_API_KEY = "test"; + } + + testChatModelInitApiKey() { + // Unset the API key env var here so this test can properly check + // the API key class arg. + process.env.GROQ_API_KEY = ""; + super.testChatModelInitApiKey(); + // Re-set the API key env var here so other tests can run properly. + process.env.GROQ_API_KEY = "test"; + } +} + +const testClass = new ChatGroqStandardUnitTests(); + +test("ChatGroqStandardUnitTests", () => { + const testResults = testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-mistralai/package.json b/libs/langchain-mistralai/package.json index 6429d3437040..762bee5fe213 100644 --- a/libs/langchain-mistralai/package.json +++ b/libs/langchain-mistralai/package.json @@ -16,11 +16,6 @@ "scripts": { "build": "yarn turbo:command build:internal --filter=@langchain/mistralai", "build:internal": "yarn lc-build:v2 --create-entrypoints --pre --tree-shaking", - "build:deps": "yarn run turbo:command build --filter=@langchain/core", - "build:esm": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist/ && rm -rf dist/tests dist/**/tests", - "build:cjs": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist-cjs/ -p tsconfig.cjs.json && yarn move-cjs-to-dist && rm -rf dist-cjs", - "build:watch": "yarn create-entrypoints && tsc --outDir dist/ --watch", - "build:scripts": "yarn create-entrypoints && yarn check-tree-shaking", "lint:eslint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/", "lint:dpdm": "dpdm --exit-code circular:1 --no-warning --no-tree src/*.ts src/**/*.ts", "lint": "yarn lint:eslint && yarn lint:dpdm", @@ -31,11 +26,11 @@ "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", "test:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:unit": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard": "yarn test:standard:unit && yarn test:standard:int", "format": "prettier --config .prettierrc --write \"src\"", - "format:check": "prettier --config .prettierrc --check \"src\"", - "move-cjs-to-dist": "yarn lc-build --config ./langchain.config.js --move-cjs-dist", - "create-entrypoints": "yarn lc-build --config ./langchain.config.js --create-entrypoints", - "check-tree-shaking": "yarn lc-build --config ./langchain.config.js --tree-shaking" + "format:check": "prettier --config .prettierrc --check \"src\"" }, "author": "LangChain", "license": "MIT", @@ -49,6 +44,7 @@ "devDependencies": { "@jest/globals": "^29.5.0", "@langchain/scripts": "~0.0.14", + "@langchain/standard-tests": "workspace:*", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "@tsconfig/recommended": "^1.0.3", diff --git a/libs/langchain-mistralai/src/chat_models.ts b/libs/langchain-mistralai/src/chat_models.ts index 4762ddf75127..a86589bd8dfd 100644 --- a/libs/langchain-mistralai/src/chat_models.ts +++ b/libs/langchain-mistralai/src/chat_models.ts @@ -82,6 +82,8 @@ interface MistralAICallOptions tool_choice?: MistralAIToolChoice; } +export interface ChatMistralAICallOptions extends MistralAICallOptions {} + /** * Input to chat model class. */ @@ -409,7 +411,7 @@ export class ChatMistralAI< this.model = this.modelName; } - protected getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { const params = this.invocationParams(options); return { ls_provider: "mistral", diff --git a/libs/langchain-mistralai/src/tests/chat_models.standard.int.test.ts b/libs/langchain-mistralai/src/tests/chat_models.standard.int.test.ts new file mode 100644 index 000000000000..0d80e46fccdb --- /dev/null +++ b/libs/langchain-mistralai/src/tests/chat_models.standard.int.test.ts @@ -0,0 +1,49 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelIntegrationTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { ChatMistralAI, ChatMistralAICallOptions } from "../chat_models.js"; + +class ChatMistralAIStandardIntegrationTests extends ChatModelIntegrationTests< + ChatMistralAICallOptions, + AIMessageChunk +> { + constructor() { + if (!process.env.MISTRAL_API_KEY) { + throw new Error( + "Can not run Mistral AI integration tests because MISTRAL_API_KEY is not set" + ); + } + super({ + Cls: ChatMistralAI, + chatModelHasToolCalling: true, + chatModelHasStructuredOutput: true, + constructorArgs: {}, + // Mistral requires function call IDs to be a-z, A-Z, 0-9, with a length of 9. + functionId: "123456789", + }); + } + + async testUsageMetadataStreaming() { + this.skipTestMessage( + "testUsageMetadataStreaming", + "ChatMistralAI", + "Streaming tokens is not currently supported." + ); + } + + async testUsageMetadata() { + this.skipTestMessage( + "testUsageMetadata", + "ChatMistralAI", + "Usage metadata tokens is not currently supported." + ); + } +} + +const testClass = new ChatMistralAIStandardIntegrationTests(); + +test("ChatMistralAIStandardIntegrationTests", async () => { + const testResults = await testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-mistralai/src/tests/chat_models.standard.test.ts b/libs/langchain-mistralai/src/tests/chat_models.standard.test.ts new file mode 100644 index 000000000000..c925f3dc6d47 --- /dev/null +++ b/libs/langchain-mistralai/src/tests/chat_models.standard.test.ts @@ -0,0 +1,53 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelUnitTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { LangSmithParams } from "@langchain/core/language_models/chat_models"; +import { ChatMistralAI, ChatMistralAICallOptions } from "../chat_models.js"; + +class ChatMistralAIStandardUnitTests extends ChatModelUnitTests< + ChatMistralAICallOptions, + AIMessageChunk +> { + constructor() { + super({ + Cls: ChatMistralAI, + chatModelHasToolCalling: true, + chatModelHasStructuredOutput: true, + constructorArgs: {}, + }); + // This must be set so method like `.bindTools` or `.withStructuredOutput` + // which we call after instantiating the model will work. + // (constructor will throw if API key is not set) + process.env.MISTRAL_API_KEY = "test"; + } + + expectedLsParams(): Partial { + console.warn( + "Overriding testStandardParams. ChatCloudflareWorkersAI does not support stop sequences." + ); + return { + ls_provider: "string", + ls_model_name: "string", + ls_model_type: "chat", + ls_temperature: 0, + ls_max_tokens: 0, + }; + } + + testChatModelInitApiKey() { + // Unset the API key env var here so this test can properly check + // the API key class arg. + process.env.MISTRAL_API_KEY = ""; + super.testChatModelInitApiKey(); + // Re-set the API key env var here so other tests can run properly. + process.env.MISTRAL_API_KEY = "test"; + } +} + +const testClass = new ChatMistralAIStandardUnitTests(); + +test("ChatMistralAIStandardUnitTests", () => { + const testResults = testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-mongodb/package.json b/libs/langchain-mongodb/package.json index 209405737203..65ef603d45c3 100644 --- a/libs/langchain-mongodb/package.json +++ b/libs/langchain-mongodb/package.json @@ -1,6 +1,6 @@ { "name": "@langchain/mongodb", - "version": "0.0.3", + "version": "0.0.4", "description": "Sample integration for LangChain.js", "type": "module", "engines": { diff --git a/libs/langchain-mongodb/src/vectorstores.ts b/libs/langchain-mongodb/src/vectorstores.ts index 5368e39f2848..3b2c208d2e3e 100644 --- a/libs/langchain-mongodb/src/vectorstores.ts +++ b/libs/langchain-mongodb/src/vectorstores.ts @@ -1,9 +1,10 @@ -import type { Collection, Document as MongoDBDocument } from "mongodb"; +import { type Collection, type Document as MongoDBDocument } from "mongodb"; import { MaxMarginalRelevanceSearchOptions, VectorStore, } from "@langchain/core/vectorstores"; import type { EmbeddingsInterface } from "@langchain/core/embeddings"; +import { chunkArray } from "@langchain/core/utils/chunk_array"; import { Document } from "@langchain/core/documents"; import { maximalMarginalRelevance } from "@langchain/core/utils/math"; import { @@ -256,6 +257,20 @@ export class MongoDBAtlasVectorSearch extends VectorStore { }); } + /** + * Delete documents from the collection + * @param ids - An array of document IDs to be deleted from the collection. + * + * @returns - A promise that resolves when all documents deleted + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + async delete(params: { ids: any[] }): Promise { + const CHUNK_SIZE = 50; + const chunkIds: any[][] = chunkArray(params.ids, CHUNK_SIZE); // eslint-disable-line @typescript-eslint/no-explicit-any + for (const chunk of chunkIds) + await this.collection.deleteMany({ _id: { $in: chunk } }); + } + /** * Static method to create an instance of MongoDBAtlasVectorSearch from a * list of texts. It first converts the texts to vectors and then adds diff --git a/libs/langchain-openai/package.json b/libs/langchain-openai/package.json index f841c46aa719..14b342daac48 100644 --- a/libs/langchain-openai/package.json +++ b/libs/langchain-openai/package.json @@ -16,26 +16,21 @@ "scripts": { "build": "yarn turbo:command build:internal --filter=@langchain/openai", "build:internal": "yarn lc-build:v2 --create-entrypoints --pre --tree-shaking", - "build:deps": "yarn run turbo:command build --filter=@langchain/core", - "build:esm": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist/", - "build:cjs": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist-cjs/ -p tsconfig.cjs.json && yarn move-cjs-to-dist && rimraf dist-cjs", - "build:watch": "yarn create-entrypoints && tsc --outDir dist/ --watch", - "build:scripts": "yarn create-entrypoints && yarn check-tree-shaking", "lint:eslint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/", "lint:dpdm": "dpdm --exit-code circular:1 --no-warning --no-tree src/*.ts src/**/*.ts", "lint": "yarn lint:eslint && yarn lint:dpdm", "lint:fix": "yarn lint:eslint --fix && yarn lint:dpdm", "clean": "rm -rf .turbo dist/", "prepack": "yarn build", - "test": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%", - "test:watch": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", - "test:single": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", + "test": "NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%", + "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", + "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", "test:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:unit": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard": "yarn test:standard:unit && yarn test:standard:int", "format": "prettier --config .prettierrc --write \"src\"", - "format:check": "prettier --config .prettierrc --check \"src\"", - "move-cjs-to-dist": "yarn lc-build --config ./langchain.config.js --move-cjs-dist", - "create-entrypoints": "yarn lc-build --config ./langchain.config.js --create-entrypoints", - "check-tree-shaking": "yarn lc-build --config ./langchain.config.js --tree-shaking" + "format:check": "prettier --config .prettierrc --check \"src\"" }, "author": "LangChain", "license": "MIT", @@ -50,6 +45,7 @@ "@azure/identity": "^4.2.0", "@jest/globals": "^29.5.0", "@langchain/scripts": "~0.0.14", + "@langchain/standard-tests": "workspace:*", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "dpdm": "^3.12.0", diff --git a/libs/langchain-openai/src/azure/chat_models.ts b/libs/langchain-openai/src/azure/chat_models.ts index 224a3e1ccf44..96c8b0ae769e 100644 --- a/libs/langchain-openai/src/azure/chat_models.ts +++ b/libs/langchain-openai/src/azure/chat_models.ts @@ -50,7 +50,7 @@ export class AzureChatOpenAI extends ChatOpenAI { super(newFields); } - protected getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { const params = super.getLsParams(options); params.ls_provider = "azure"; return params; diff --git a/libs/langchain-openai/src/chat_models.ts b/libs/langchain-openai/src/chat_models.ts index f68fa5fdce72..78ace7e3d13c 100644 --- a/libs/langchain-openai/src/chat_models.ts +++ b/libs/langchain-openai/src/chat_models.ts @@ -491,7 +491,7 @@ export class ChatOpenAI< }; } - protected getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { const params = this.invocationParams(options); return { ls_provider: "openai", diff --git a/libs/langchain-openai/src/tests/azure/chat_models.standard.int.test.ts b/libs/langchain-openai/src/tests/azure/chat_models.standard.int.test.ts new file mode 100644 index 000000000000..cfec5cfe8eb8 --- /dev/null +++ b/libs/langchain-openai/src/tests/azure/chat_models.standard.int.test.ts @@ -0,0 +1,45 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelIntegrationTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { AzureChatOpenAI } from "../../azure/chat_models.js"; +import { ChatOpenAICallOptions } from "../../chat_models.js"; + +class AzureChatOpenAIStandardIntegrationTests extends ChatModelIntegrationTests< + ChatOpenAICallOptions, + AIMessageChunk +> { + constructor() { + super({ + Cls: AzureChatOpenAI, + chatModelHasToolCalling: true, + chatModelHasStructuredOutput: true, + constructorArgs: { + model: "gpt-3.5-turbo", + }, + }); + } + + async testToolMessageHistoriesListContent() { + this.skipTestMessage( + "testToolMessageHistoriesListContent", + "AzureChatOpenAI", + "Not properly implemented." + ); + } + + async testUsageMetadataStreaming() { + this.skipTestMessage( + "testUsageMetadataStreaming", + "AzureChatOpenAI", + "Streaming tokens is not currently supported." + ); + } +} + +const testClass = new AzureChatOpenAIStandardIntegrationTests(); + +test("AzureChatOpenAIStandardIntegrationTests", async () => { + const testResults = await testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-openai/src/tests/azure/chat_models.standard.test.ts b/libs/langchain-openai/src/tests/azure/chat_models.standard.test.ts new file mode 100644 index 000000000000..1dd91e128e1b --- /dev/null +++ b/libs/langchain-openai/src/tests/azure/chat_models.standard.test.ts @@ -0,0 +1,37 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelUnitTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { AzureChatOpenAI } from "../../azure/chat_models.js"; +import { ChatOpenAICallOptions } from "../../chat_models.js"; + +class AzureChatOpenAIStandardUnitTests extends ChatModelUnitTests< + ChatOpenAICallOptions, + AIMessageChunk +> { + constructor() { + super({ + Cls: AzureChatOpenAI, + chatModelHasToolCalling: true, + chatModelHasStructuredOutput: true, + constructorArgs: {}, + }); + process.env.AZURE_OPENAI_API_KEY = "test"; + process.env.AZURE_OPENAI_API_DEPLOYMENT_NAME = "test"; + process.env.AZURE_OPENAI_API_VERSION = "test"; + process.env.AZURE_OPENAI_BASE_PATH = "test"; + } + + testChatModelInitApiKey() { + console.warn( + "AzureChatOpenAI does not require a single API key. Skipping..." + ); + } +} + +const testClass = new AzureChatOpenAIStandardUnitTests(); + +test("AzureChatOpenAIStandardUnitTests", () => { + const testResults = testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-openai/src/tests/chat_models.standard.int.test.ts b/libs/langchain-openai/src/tests/chat_models.standard.int.test.ts new file mode 100644 index 000000000000..9c0c4e0d90d2 --- /dev/null +++ b/libs/langchain-openai/src/tests/chat_models.standard.int.test.ts @@ -0,0 +1,51 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelIntegrationTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { ChatOpenAI, ChatOpenAICallOptions } from "../chat_models.js"; + +class ChatOpenAIStandardIntegrationTests extends ChatModelIntegrationTests< + ChatOpenAICallOptions, + AIMessageChunk +> { + constructor() { + if (!process.env.OPENAI_API_KEY) { + throw new Error( + "OPENAI_API_KEY must be set to run standard integration tests." + ); + } + super({ + Cls: ChatOpenAI, + chatModelHasToolCalling: true, + chatModelHasStructuredOutput: true, + constructorArgs: { + model: "gpt-3.5-turbo", + }, + }); + } + + async testToolMessageHistoriesListContent() { + console.warn( + "ChatOpenAI testToolMessageHistoriesListContent test known failure. Skipping..." + ); + } + + async testUsageMetadataStreaming() { + // ChatOpenAI does not support streaming tokens by + // default, so we must pass in a call option to + // enable streaming tokens. + const callOptions: ChatOpenAI["ParsedCallOptions"] = { + stream_options: { + include_usage: true, + }, + }; + await super.testUsageMetadataStreaming(callOptions); + } +} + +const testClass = new ChatOpenAIStandardIntegrationTests(); + +test("ChatOpenAIStandardIntegrationTests", async () => { + const testResults = await testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-openai/src/tests/chat_models.standard.test.ts b/libs/langchain-openai/src/tests/chat_models.standard.test.ts new file mode 100644 index 000000000000..7b794cdccf1e --- /dev/null +++ b/libs/langchain-openai/src/tests/chat_models.standard.test.ts @@ -0,0 +1,39 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelUnitTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { ChatOpenAI, ChatOpenAICallOptions } from "../chat_models.js"; + +class ChatOpenAIStandardUnitTests extends ChatModelUnitTests< + ChatOpenAICallOptions, + AIMessageChunk +> { + constructor() { + super({ + Cls: ChatOpenAI, + chatModelHasToolCalling: true, + chatModelHasStructuredOutput: true, + constructorArgs: {}, + }); + // This must be set so method like `.bindTools` or `.withStructuredOutput` + // which we call after instantiating the model will work. + // (constructor will throw if API key is not set) + process.env.OPENAI_API_KEY = "test"; + } + + testChatModelInitApiKey() { + // Unset the API key env var here so this test can properly check + // the API key class arg. + process.env.OPENAI_API_KEY = ""; + super.testChatModelInitApiKey(); + // Re-set the API key env var here so other tests can run properly. + process.env.OPENAI_API_KEY = "test"; + } +} + +const testClass = new ChatOpenAIStandardUnitTests(); + +test("ChatOpenAIStandardUnitTests", () => { + const testResults = testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-standard-tests/.eslintrc.cjs b/libs/langchain-standard-tests/.eslintrc.cjs new file mode 100644 index 000000000000..51a16939ff9f --- /dev/null +++ b/libs/langchain-standard-tests/.eslintrc.cjs @@ -0,0 +1,67 @@ +module.exports = { + extends: [ + "airbnb-base", + "eslint:recommended", + "prettier", + "plugin:@typescript-eslint/recommended", + ], + parserOptions: { + ecmaVersion: 12, + parser: "@typescript-eslint/parser", + project: "./tsconfig.json", + sourceType: "module", + }, + plugins: ["@typescript-eslint", "no-instanceof"], + ignorePatterns: [ + ".eslintrc.cjs", + "scripts", + "node_modules", + "dist", + "dist-cjs", + "*.js", + "*.cjs", + "*.d.ts", + ], + rules: { + "no-process-env": 2, + "no-instanceof/no-instanceof": 2, + "@typescript-eslint/explicit-module-boundary-types": 0, + "@typescript-eslint/no-empty-function": 0, + "@typescript-eslint/no-shadow": 0, + "@typescript-eslint/no-empty-interface": 0, + "@typescript-eslint/no-explicit-any": "warn", + "@typescript-eslint/no-use-before-define": ["error", "nofunc"], + "@typescript-eslint/no-unused-vars": ["warn", { args: "none" }], + "@typescript-eslint/no-floating-promises": "error", + "@typescript-eslint/no-misused-promises": "error", + camelcase: 0, + "class-methods-use-this": 0, + "import/extensions": [2, "ignorePackages"], + "import/no-extraneous-dependencies": [ + "error", + { devDependencies: ["**/*.test.ts"] }, + ], + "import/no-unresolved": 0, + "import/prefer-default-export": 0, + "keyword-spacing": "error", + "max-classes-per-file": 0, + "max-len": 0, + "no-await-in-loop": 0, + "no-bitwise": 0, + "no-console": 0, + "no-restricted-syntax": 0, + "no-shadow": 0, + "no-continue": 0, + "no-void": 0, + "no-underscore-dangle": 0, + "no-use-before-define": 0, + "no-useless-constructor": 0, + "no-return-await": 0, + "consistent-return": 0, + "no-else-return": 0, + "func-names": 0, + "no-lonely-if": 0, + "prefer-rest-params": 0, + "new-cap": ["error", { properties: false, capIsNew: false }], + }, +}; diff --git a/libs/langchain-standard-tests/.gitignore b/libs/langchain-standard-tests/.gitignore new file mode 100644 index 000000000000..c10034e2f1be --- /dev/null +++ b/libs/langchain-standard-tests/.gitignore @@ -0,0 +1,7 @@ +index.cjs +index.js +index.d.ts +index.d.cts +node_modules +dist +.yarn diff --git a/libs/langchain-standard-tests/.prettierrc b/libs/langchain-standard-tests/.prettierrc new file mode 100644 index 000000000000..ba08ff04f677 --- /dev/null +++ b/libs/langchain-standard-tests/.prettierrc @@ -0,0 +1,19 @@ +{ + "$schema": "https://json.schemastore.org/prettierrc", + "printWidth": 80, + "tabWidth": 2, + "useTabs": false, + "semi": true, + "singleQuote": false, + "quoteProps": "as-needed", + "jsxSingleQuote": false, + "trailingComma": "es5", + "bracketSpacing": true, + "arrowParens": "always", + "requirePragma": false, + "insertPragma": false, + "proseWrap": "preserve", + "htmlWhitespaceSensitivity": "css", + "vueIndentScriptAndStyle": false, + "endOfLine": "lf" +} diff --git a/libs/langchain-standard-tests/.release-it.json b/libs/langchain-standard-tests/.release-it.json new file mode 100644 index 000000000000..522ee6abf705 --- /dev/null +++ b/libs/langchain-standard-tests/.release-it.json @@ -0,0 +1,10 @@ +{ + "github": { + "release": true, + "autoGenerate": true, + "tokenRef": "GITHUB_TOKEN_RELEASE" + }, + "npm": { + "versionArgs": ["--workspaces-update=false"] + } +} diff --git a/libs/langchain-standard-tests/LICENSE b/libs/langchain-standard-tests/LICENSE new file mode 100644 index 000000000000..8cd8f501eb49 --- /dev/null +++ b/libs/langchain-standard-tests/LICENSE @@ -0,0 +1,21 @@ +The MIT License + +Copyright (c) 2023 LangChain + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/libs/langchain-standard-tests/README.md b/libs/langchain-standard-tests/README.md new file mode 100644 index 000000000000..5b2b678af672 --- /dev/null +++ b/libs/langchain-standard-tests/README.md @@ -0,0 +1,94 @@ +# LangChain.js Standard Tests + +This package contains the base standard tests for LangChain.js. It includes unit, and integration test classes. +This package is not intended to be used outside of the LangChain.js project, and thus it is not published to npm. + +At the moment, we only have support for standard tets for chat models. + +## Usage + +Each LangChain.js integration should contain both unit and integration standard tests. +The package should have `@langchain/standard-tests` as a dev workspace dependency like so: + +`package.json`: +```json +{ + "devDependencies": { + "@langchain/standard-tests": "workspace:*" + } +} +``` + +To use the standard tests, you could create two files: + +- `src/tests/chat_models.standard.test.ts` - chat model unit tests +- `src/tests/chat_models.standard.int.test.ts` - chat model integration tests + +Your unit test file should look like this: + +`chat_models.standard.test.ts`: +```typescript +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelUnitTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { MyChatModel, MyChatModelCallOptions } from "../chat_models.js"; + +class MyChatModelStandardUnitTests extends ChatModelUnitTests< + MyChatModelCallOptions, + AIMessageChunk +> { + constructor() { + super({ + Cls: MyChatModel, + chatModelHasToolCalling: true, // Set to true if the model has tool calling support + chatModelHasStructuredOutput: true, // Set to true if the model has withStructuredOutput support + constructorArgs: {}, // Any additional constructor args + }); + // This must be set so method like `.bindTools` or `.withStructuredOutput` + // which we call after instantiating the model will work. + // (constructor will throw if API key is not set) + process.env.CHAT_MODEL_API_KEY = "test"; + } + + testChatModelInitApiKey() { + // Unset the API key env var here so this test can properly check + // the API key class arg. + process.env.CHAT_MODEL_API_KEY = ""; + super.testChatModelInitApiKey(); + // Re-set the API key env var here so other tests can run properly. + process.env.CHAT_MODEL_API_KEY = "test"; + } +} + +const testClass = new MyChatModelStandardUnitTests(); + +test("MyChatModelStandardUnitTests", () => { + const testResults = testClass.runTests(); + expect(testResults).toBe(true); +}); +``` + +To use the standard tests, extend the `ChatModelUnitTests` class, passing in your chat model's call options and message chunk types. Super the constructor with your chat model class, any additional constructor args, and set `chatModelHasToolCalling` and `chatModelHasStructuredOutput` flags if supported. + +Set the model env var in the constructor directly to `process.env` for the tests to run properly. You can optionally override test methods to replace or add code before/after the test runs. + +Run all tests by calling `.runTests()`, which returns `true` if all tests pass, `false` otherwise. Tests are called in `try`/`catch` blocks, so failing tests are caught and marked as failed, but the rest still run. + +For integration tests, extend `ChatModelIntegrationTests` instead. Integration tests have an optional arg for all methods (except `withStructuredOutput`) to pass in "invoke" time call options. For example, in the OpenAI integration test: + +```typescript +async testUsageMetadataStreaming() { + // ChatOpenAI does not support streaming tokens by + // default, so we must pass in a call option to + // enable streaming tokens. + const callOptions: ChatOpenAI["ParsedCallOptions"] = { + stream_options: { + include_usage: true, + }, + }; + await super.testUsageMetadataStreaming(callOptions); +} +``` + +This overrides the base `testUsageMetadataStreaming` to pass a `callOptions` arg enabling streaming tokens. diff --git a/libs/langchain-standard-tests/jest.config.cjs b/libs/langchain-standard-tests/jest.config.cjs new file mode 100644 index 000000000000..994826496bc5 --- /dev/null +++ b/libs/langchain-standard-tests/jest.config.cjs @@ -0,0 +1,21 @@ +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: "ts-jest/presets/default-esm", + testEnvironment: "./jest.env.cjs", + modulePathIgnorePatterns: ["dist/", "docs/"], + moduleNameMapper: { + "^(\\.{1,2}/.*)\\.js$": "$1", + }, + transform: { + "^.+\\.tsx?$": ["@swc/jest"], + }, + transformIgnorePatterns: [ + "/node_modules/", + "\\.pnp\\.[^\\/]+$", + "./scripts/jest-setup-after-env.js", + ], + setupFiles: ["dotenv/config"], + testTimeout: 20_000, + passWithNoTests: true, + collectCoverageFrom: ["src/**/*.ts"], +}; diff --git a/libs/langchain-standard-tests/jest.env.cjs b/libs/langchain-standard-tests/jest.env.cjs new file mode 100644 index 000000000000..2ccedccb8672 --- /dev/null +++ b/libs/langchain-standard-tests/jest.env.cjs @@ -0,0 +1,12 @@ +const { TestEnvironment } = require("jest-environment-node"); + +class AdjustedTestEnvironmentToSupportFloat32Array extends TestEnvironment { + constructor(config, context) { + // Make `instanceof Float32Array` return true in tests + // to avoid https://github.com/xenova/transformers.js/issues/57 and https://github.com/jestjs/jest/issues/2549 + super(config, context); + this.global.Float32Array = Float32Array; + } +} + +module.exports = AdjustedTestEnvironmentToSupportFloat32Array; diff --git a/libs/langchain-standard-tests/langchain.config.js b/libs/langchain-standard-tests/langchain.config.js new file mode 100644 index 000000000000..25a18707a316 --- /dev/null +++ b/libs/langchain-standard-tests/langchain.config.js @@ -0,0 +1,21 @@ +import { resolve, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +/** + * @param {string} relativePath + * @returns {string} + */ +function abs(relativePath) { + return resolve(dirname(fileURLToPath(import.meta.url)), relativePath); +} + +export const config = { + internals: [/node\:/, /@langchain\/core\//], + entrypoints: { + index: "index", + }, + tsConfigPath: resolve("./tsconfig.json"), + cjsSource: "./dist-cjs", + cjsDestination: "./dist", + abs, +}; diff --git a/libs/langchain-standard-tests/package.json b/libs/langchain-standard-tests/package.json new file mode 100644 index 000000000000..794199bf70be --- /dev/null +++ b/libs/langchain-standard-tests/package.json @@ -0,0 +1,84 @@ +{ + "name": "@langchain/standard-tests", + "version": "0.0.0", + "description": "Standard tests for LangChain.js", + "type": "module", + "engines": { + "node": ">=18" + }, + "main": "./index.js", + "types": "./index.d.ts", + "repository": { + "type": "git", + "url": "git@github.com:langchain-ai/langchainjs.git" + }, + "homepage": "https://github.com/langchain-ai/langchainjs/tree/main/libs/langchain-standard-tests/", + "scripts": { + "build": "yarn turbo:command build:internal --filter=@langchain/standard-tests", + "build:internal": "yarn lc-build:v2 --create-entrypoints --pre --tree-shaking", + "lint:eslint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/", + "lint:dpdm": "dpdm --exit-code circular:1 --no-warning --no-tree src/*.ts src/**/*.ts", + "lint": "yarn lint:eslint && yarn lint:dpdm", + "lint:fix": "yarn lint:eslint --fix && yarn lint:dpdm", + "clean": "rm -rf .turbo dist/", + "prepack": "yarn build", + "test": "NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%", + "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", + "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", + "test:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "format": "prettier --config .prettierrc --write \"src\"", + "format:check": "prettier --config .prettierrc --check \"src\"" + }, + "author": "LangChain", + "license": "MIT", + "dependencies": { + "@jest/globals": "^29.5.0", + "@langchain/core": "workspace:*", + "zod": "^3.22.4" + }, + "devDependencies": { + "@langchain/scripts": "workspace:*", + "@swc/core": "^1.3.90", + "@swc/jest": "^0.2.29", + "@tsconfig/recommended": "^1.0.3", + "@typescript-eslint/eslint-plugin": "^6.12.0", + "@typescript-eslint/parser": "^6.12.0", + "dotenv": "^16.3.1", + "dpdm": "^3.12.0", + "eslint": "^8.33.0", + "eslint-config-airbnb-base": "^15.0.0", + "eslint-config-prettier": "^8.6.0", + "eslint-plugin-import": "^2.27.5", + "eslint-plugin-no-instanceof": "^1.0.1", + "eslint-plugin-prettier": "^4.2.1", + "jest": "^29.5.0", + "jest-environment-node": "^29.6.4", + "prettier": "^2.8.3", + "release-it": "^15.10.1", + "rollup": "^4.5.2", + "ts-jest": "^29.1.0", + "typescript": "^5.4.5" + }, + "publishConfig": { + "access": "public" + }, + "exports": { + ".": { + "types": { + "import": "./index.d.ts", + "require": "./index.d.cts", + "default": "./index.d.ts" + }, + "import": "./index.js", + "require": "./index.cjs" + }, + "./package.json": "./package.json" + }, + "files": [ + "dist/", + "index.cjs", + "index.js", + "index.d.ts", + "index.d.cts" + ] +} diff --git a/libs/langchain-standard-tests/scripts/jest-setup-after-env.js b/libs/langchain-standard-tests/scripts/jest-setup-after-env.js new file mode 100644 index 000000000000..778cf7437a20 --- /dev/null +++ b/libs/langchain-standard-tests/scripts/jest-setup-after-env.js @@ -0,0 +1,3 @@ +import { awaitAllCallbacks } from "@langchain/core/callbacks/promises"; + +afterAll(awaitAllCallbacks); diff --git a/libs/langchain-standard-tests/src/base.ts b/libs/langchain-standard-tests/src/base.ts new file mode 100644 index 000000000000..fa4d56404795 --- /dev/null +++ b/libs/langchain-standard-tests/src/base.ts @@ -0,0 +1,85 @@ +import { + BaseChatModel, + BaseChatModelCallOptions, +} from "@langchain/core/language_models/chat_models"; +import { BaseMessageChunk } from "@langchain/core/messages"; + +export type RecordStringAny = Record; + +export type BaseChatModelConstructor< + CallOptions extends BaseChatModelCallOptions = BaseChatModelCallOptions, + OutputMessageType extends BaseMessageChunk = BaseMessageChunk, + ConstructorArgs extends RecordStringAny = RecordStringAny +> = new (args: ConstructorArgs) => BaseChatModel< + CallOptions, + OutputMessageType +>; + +export type BaseChatModelsTestsFields< + CallOptions extends BaseChatModelCallOptions = BaseChatModelCallOptions, + OutputMessageType extends BaseMessageChunk = BaseMessageChunk, + ConstructorArgs extends RecordStringAny = RecordStringAny +> = { + Cls: BaseChatModelConstructor< + CallOptions, + OutputMessageType, + ConstructorArgs + >; + chatModelHasToolCalling: boolean; + chatModelHasStructuredOutput: boolean; + constructorArgs: ConstructorArgs; +}; + +export class BaseChatModelsTests< + CallOptions extends BaseChatModelCallOptions = BaseChatModelCallOptions, + OutputMessageType extends BaseMessageChunk = BaseMessageChunk, + ConstructorArgs extends RecordStringAny = RecordStringAny +> implements + BaseChatModelsTestsFields +{ + Cls: BaseChatModelConstructor< + CallOptions, + OutputMessageType, + ConstructorArgs + >; + + chatModelHasToolCalling: boolean; + + chatModelHasStructuredOutput: boolean; + + constructorArgs: ConstructorArgs; + + constructor( + fields: BaseChatModelsTestsFields< + CallOptions, + OutputMessageType, + ConstructorArgs + > + ) { + this.Cls = fields.Cls; + this.chatModelHasToolCalling = fields.chatModelHasToolCalling; + this.chatModelHasStructuredOutput = fields.chatModelHasStructuredOutput; + this.constructorArgs = fields.constructorArgs; + } + + get multipleApiKeysRequiredMessage(): string { + return "Multiple API keys are required."; + } + + /** + * Log a warning message when skipping a test. + */ + skipTestMessage( + testName: string, + chatClassName: string, + extra?: string + ): void { + console.warn( + { + chatClassName, + reason: extra ?? "n/a", + }, + `Skipping ${testName}.` + ); + } +} diff --git a/libs/langchain-standard-tests/src/index.ts b/libs/langchain-standard-tests/src/index.ts new file mode 100644 index 000000000000..57ff0dc1b045 --- /dev/null +++ b/libs/langchain-standard-tests/src/index.ts @@ -0,0 +1,2 @@ +export * from "./unit_tests/chat_models.js"; +export * from "./integration_tests/chat_models.js"; diff --git a/libs/langchain-standard-tests/src/integration_tests/chat_models.ts b/libs/langchain-standard-tests/src/integration_tests/chat_models.ts new file mode 100644 index 000000000000..bb249bb30364 --- /dev/null +++ b/libs/langchain-standard-tests/src/integration_tests/chat_models.ts @@ -0,0 +1,456 @@ +import { expect } from "@jest/globals"; +import { BaseChatModelCallOptions } from "@langchain/core/language_models/chat_models"; +import { + AIMessage, + AIMessageChunk, + BaseMessageChunk, + HumanMessage, + ToolMessage, + UsageMetadata, +} from "@langchain/core/messages"; +import { z } from "zod"; +import { StructuredTool } from "@langchain/core/tools"; +import { + BaseChatModelsTests, + BaseChatModelsTestsFields, + RecordStringAny, +} from "../base.js"; + +const adderSchema = /* #__PURE__ */ z + .object({ + a: z.number().int().describe("The first integer to add."), + b: z.number().int().describe("The second integer to add."), + }) + .describe("Add two integers"); + +class AdderTool extends StructuredTool { + name = "AdderTool"; + + description = adderSchema.description ?? "description"; + + schema = adderSchema; + + async _call(input: z.infer) { + const sum = input.a + input.b; + return JSON.stringify({ result: sum }); + } +} + +export abstract class ChatModelIntegrationTests< + CallOptions extends BaseChatModelCallOptions = BaseChatModelCallOptions, + OutputMessageType extends BaseMessageChunk = BaseMessageChunk, + ConstructorArgs extends RecordStringAny = RecordStringAny +> extends BaseChatModelsTests { + functionId = "abc123"; + + constructor( + fields: BaseChatModelsTestsFields< + CallOptions, + OutputMessageType, + ConstructorArgs + > & { + /** + * The ID to set for function calls. + * Set this field to override the default function ID. + * @default "abc123" + */ + functionId?: string; + } + ) { + super(fields); + this.functionId = fields.functionId ?? this.functionId; + } + + async testInvoke( + callOptions?: InstanceType["ParsedCallOptions"] + ) { + const chatModel = new this.Cls(this.constructorArgs); + const result = await chatModel.invoke("Hello", callOptions); + expect(result).toBeDefined(); + expect(result).toBeInstanceOf(AIMessage); + expect(typeof result.content).toBe("string"); + expect(result.content.length).toBeGreaterThan(0); + } + + async testStream( + callOptions?: InstanceType["ParsedCallOptions"] + ) { + const chatModel = new this.Cls(this.constructorArgs); + let numChars = 0; + + for await (const token of await chatModel.stream("Hello", callOptions)) { + expect(token).toBeDefined(); + expect(token).toBeInstanceOf(AIMessageChunk); + expect(typeof token.content).toBe("string"); + numChars += token.content.length; + } + + expect(numChars).toBeGreaterThan(0); + } + + async testBatch( + callOptions?: InstanceType["ParsedCallOptions"] + ) { + const chatModel = new this.Cls(this.constructorArgs); + const batchResults = await chatModel.batch(["Hello", "Hey"], callOptions); + expect(batchResults).toBeDefined(); + expect(Array.isArray(batchResults)).toBe(true); + expect(batchResults.length).toBe(2); + for (const result of batchResults) { + expect(result).toBeDefined(); + expect(result).toBeInstanceOf(AIMessage); + expect(typeof result.content).toBe("string"); + expect(result.content.length).toBeGreaterThan(0); + } + } + + async testConversation( + callOptions?: InstanceType["ParsedCallOptions"] + ) { + const chatModel = new this.Cls(this.constructorArgs); + const messages = [ + new HumanMessage("hello"), + new AIMessage("hello"), + new HumanMessage("how are you"), + ]; + const result = await chatModel.invoke(messages, callOptions); + expect(result).toBeDefined(); + expect(result).toBeInstanceOf(AIMessage); + expect(typeof result.content).toBe("string"); + expect(result.content.length).toBeGreaterThan(0); + } + + async testUsageMetadata( + callOptions?: InstanceType["ParsedCallOptions"] + ) { + const chatModel = new this.Cls(this.constructorArgs); + const result = await chatModel.invoke("Hello", callOptions); + expect(result).toBeDefined(); + expect(result).toBeInstanceOf(AIMessage); + if (!("usage_metadata" in result)) { + throw new Error("result is not an instance of AIMessage"); + } + const usageMetadata = result.usage_metadata as UsageMetadata; + expect(usageMetadata).toBeDefined(); + expect(typeof usageMetadata.input_tokens).toBe("number"); + expect(typeof usageMetadata.output_tokens).toBe("number"); + expect(typeof usageMetadata.total_tokens).toBe("number"); + } + + async testUsageMetadataStreaming( + callOptions?: InstanceType["ParsedCallOptions"] + ) { + const chatModel = new this.Cls(this.constructorArgs); + let finalChunks: AIMessageChunk | undefined; + for await (const chunk of await chatModel.stream("Hello", callOptions)) { + expect(chunk).toBeDefined(); + expect(chunk).toBeInstanceOf(AIMessageChunk); + if (!finalChunks) { + finalChunks = chunk; + } else { + finalChunks = finalChunks.concat(chunk); + } + } + if (!finalChunks) { + throw new Error("finalChunks is undefined"); + } + const usageMetadata = finalChunks.usage_metadata; + expect(usageMetadata).toBeDefined(); + if (!usageMetadata) { + throw new Error("usageMetadata is undefined"); + } + expect(typeof usageMetadata.input_tokens).toBe("number"); + expect(typeof usageMetadata.output_tokens).toBe("number"); + expect(typeof usageMetadata.total_tokens).toBe("number"); + } + + /** + * Test that message histories are compatible with string tool contents + * (e.g. OpenAI). + * @returns {Promise} + */ + async testToolMessageHistoriesStringContent( + callOptions?: InstanceType["ParsedCallOptions"] + ) { + if (!this.chatModelHasToolCalling) { + console.log("Test requires tool calling. Skipping..."); + return; + } + + const model = new this.Cls(this.constructorArgs); + const adderTool = new AdderTool(); + if (!model.bindTools) { + throw new Error( + "bindTools undefined. Cannot test tool message histories." + ); + } + const modelWithTools = model.bindTools([adderTool]); + const functionName = adderTool.name; + const functionArgs = { a: 1, b: 2 }; + + const { functionId } = this; + const functionResult = await adderTool.invoke(functionArgs); + + const messagesStringContent = [ + new HumanMessage("What is 1 + 2"), + // string content (e.g. OpenAI) + new AIMessage({ + content: "", + tool_calls: [ + { + name: functionName, + args: functionArgs, + id: functionId, + }, + ], + }), + new ToolMessage(functionResult, functionId, functionName), + ]; + + const resultStringContent = await modelWithTools.invoke( + messagesStringContent, + callOptions + ); + expect(resultStringContent).toBeInstanceOf(AIMessage); + } + + /** + * Test that message histories are compatible with list tool contents + * (e.g. Anthropic). + * @returns {Promise} + */ + async testToolMessageHistoriesListContent( + callOptions?: InstanceType["ParsedCallOptions"] + ) { + if (!this.chatModelHasToolCalling) { + console.log("Test requires tool calling. Skipping..."); + return; + } + + const model = new this.Cls(this.constructorArgs); + const adderTool = new AdderTool(); + if (!model.bindTools) { + throw new Error( + "bindTools undefined. Cannot test tool message histories." + ); + } + const modelWithTools = model.bindTools([adderTool]); + const functionName = adderTool.name; + const functionArgs = { a: 1, b: 2 }; + + const { functionId } = this; + const functionResult = await adderTool.invoke(functionArgs); + + const messagesListContent = [ + new HumanMessage("What is 1 + 2"), + // List content (e.g., Anthropic) + new AIMessage({ + content: [ + { type: "text", text: "some text" }, + { + type: "tool_use", + id: functionId, + name: functionName, + input: functionArgs, + }, + ], + tool_calls: [ + { + name: functionName, + args: functionArgs, + id: functionId, + }, + ], + }), + new ToolMessage(functionResult, functionId, functionName), + ]; + + const resultListContent = await modelWithTools.invoke( + messagesListContent, + callOptions + ); + expect(resultListContent).toBeInstanceOf(AIMessage); + } + + /** + * Test that model can process few-shot examples with tool calls. + * @returns {Promise} + */ + async testStructuredFewShotExamples( + callOptions?: InstanceType["ParsedCallOptions"] + ) { + if (!this.chatModelHasToolCalling) { + console.log("Test requires tool calling. Skipping..."); + return; + } + + const model = new this.Cls(this.constructorArgs); + const adderTool = new AdderTool(); + if (!model.bindTools) { + throw new Error("bindTools undefined. Cannot test few-shot examples."); + } + const modelWithTools = model.bindTools([adderTool]); + const functionName = adderTool.name; + const functionArgs = { a: 1, b: 2 }; + + const { functionId } = this; + const functionResult = await adderTool.invoke(functionArgs); + + const messagesStringContent = [ + new HumanMessage("What is 1 + 2"), + new AIMessage({ + content: "", + tool_calls: [ + { + name: functionName, + args: functionArgs, + id: functionId, + }, + ], + }), + new ToolMessage(functionResult, functionId, functionName), + new AIMessage(functionResult), + new HumanMessage("What is 3 + 4"), + ]; + + const resultStringContent = await modelWithTools.invoke( + messagesStringContent, + callOptions + ); + expect(resultStringContent).toBeInstanceOf(AIMessage); + } + + async testWithStructuredOutput() { + if (!this.chatModelHasStructuredOutput) { + console.log("Test requires withStructuredOutput. Skipping..."); + return; + } + + const model = new this.Cls(this.constructorArgs); + if (!model.withStructuredOutput) { + throw new Error( + "withStructuredOutput undefined. Cannot test tool message histories." + ); + } + const modelWithTools = model.withStructuredOutput(adderSchema); + + const resultStringContent = await modelWithTools.invoke("What is 1 + 2"); + expect(resultStringContent.a).toBeDefined(); + expect([1, 2].includes(resultStringContent.a)).toBeTruthy(); + expect(resultStringContent.b).toBeDefined(); + expect([1, 2].includes(resultStringContent.b)).toBeTruthy(); + } + + async testWithStructuredOutputIncludeRaw() { + if (!this.chatModelHasStructuredOutput) { + console.log("Test requires withStructuredOutput. Skipping..."); + return; + } + + const model = new this.Cls(this.constructorArgs); + if (!model.withStructuredOutput) { + throw new Error( + "withStructuredOutput undefined. Cannot test tool message histories." + ); + } + const modelWithTools = model.withStructuredOutput(adderSchema, { + includeRaw: true, + }); + + const resultStringContent = await modelWithTools.invoke("What is 1 + 2"); + expect(resultStringContent.raw).toBeInstanceOf(AIMessage); + expect(resultStringContent.parsed.a).toBeDefined(); + expect([1, 2].includes(resultStringContent.parsed.a)).toBeTruthy(); + expect(resultStringContent.parsed.b).toBeDefined(); + expect([1, 2].includes(resultStringContent.parsed.b)).toBeTruthy(); + } + + /** + * Run all unit tests for the chat model. + * Each test is wrapped in a try/catch block to prevent the entire test suite from failing. + * If a test fails, the error is logged to the console, and the test suite continues. + * @returns {boolean} + */ + async runTests(): Promise { + let allTestsPassed = true; + + try { + await this.testInvoke(); + } catch (e: any) { + allTestsPassed = false; + console.error("testInvoke failed", e); + } + + try { + await this.testStream(); + } catch (e: any) { + allTestsPassed = false; + console.error("testStream failed", e); + } + + try { + await this.testBatch(); + } catch (e: any) { + allTestsPassed = false; + console.error("testBatch failed", e); + } + + try { + await this.testConversation(); + } catch (e: any) { + allTestsPassed = false; + console.error("testConversation failed", e); + } + + try { + await this.testUsageMetadata(); + } catch (e: any) { + allTestsPassed = false; + console.error("testUsageMetadata failed", e); + } + + try { + await this.testUsageMetadataStreaming(); + } catch (e: any) { + allTestsPassed = false; + console.error("testUsageMetadataStreaming failed", e); + } + + try { + await this.testToolMessageHistoriesStringContent(); + } catch (e: any) { + allTestsPassed = false; + console.error("testToolMessageHistoriesStringContent failed", e); + } + + try { + await this.testToolMessageHistoriesListContent(); + } catch (e: any) { + allTestsPassed = false; + console.error("testToolMessageHistoriesListContent failed", e); + } + + try { + await this.testStructuredFewShotExamples(); + } catch (e: any) { + allTestsPassed = false; + console.error("testStructuredFewShotExamples failed", e); + } + + try { + await this.testWithStructuredOutput(); + } catch (e: any) { + allTestsPassed = false; + console.error("testWithStructuredOutput failed", e); + } + + try { + await this.testWithStructuredOutputIncludeRaw(); + } catch (e: any) { + allTestsPassed = false; + console.error("testWithStructuredOutputIncludeRaw failed", e); + } + + return allTestsPassed; + } +} diff --git a/libs/langchain-standard-tests/src/unit_tests/chat_models.ts b/libs/langchain-standard-tests/src/unit_tests/chat_models.ts new file mode 100644 index 000000000000..47de9478fcc2 --- /dev/null +++ b/libs/langchain-standard-tests/src/unit_tests/chat_models.ts @@ -0,0 +1,174 @@ +import { expect } from "@jest/globals"; +import { + BaseChatModelCallOptions, + LangSmithParams, +} from "@langchain/core/language_models/chat_models"; +import { BaseMessageChunk } from "@langchain/core/messages"; +import { z } from "zod"; +import { StructuredTool } from "@langchain/core/tools"; +import { + BaseChatModelsTests, + BaseChatModelsTestsFields, + RecordStringAny, +} from "../base.js"; + +const person = /* #__PURE__ */ z + .object({ + name: z.string().describe("Name of the person"), + age: z.number().int().positive().describe("Age of the person"), + }) + .describe("A person"); + +class PersonTool extends StructuredTool { + name = "PersonTool"; + + description = person.description ?? "description"; + + schema = person; + + async _call(input: z.infer) { + return JSON.stringify(input); + } +} + +export abstract class ChatModelUnitTests< + CallOptions extends BaseChatModelCallOptions = BaseChatModelCallOptions, + OutputMessageType extends BaseMessageChunk = BaseMessageChunk, + ConstructorArgs extends RecordStringAny = RecordStringAny +> extends BaseChatModelsTests { + constructor( + fields: BaseChatModelsTestsFields< + CallOptions, + OutputMessageType, + ConstructorArgs + > + ) { + const standardChatModelParams: RecordStringAny = { + temperature: 0, + maxTokens: 100, + timeout: 60, + stopSequences: [], + maxRetries: 2, + }; + super({ + ...fields, + constructorArgs: { + ...standardChatModelParams, + ...fields.constructorArgs, + }, + }); + } + + /** + * Override this method if the chat model being tested does not + * support all expected LangSmith parameters. + * @returns {Partial} The LangSmith parameters expected by the chat model. + */ + expectedLsParams(): Partial { + return { + ls_provider: "string", + ls_model_name: "string", + ls_model_type: "chat", + ls_temperature: 0, + ls_max_tokens: 0, + ls_stop: ["Array"], + }; + } + + testChatModelInit() { + const chatModel = new this.Cls(this.constructorArgs); + expect(chatModel).toBeDefined(); + } + + testChatModelInitApiKey() { + const params = { ...this.constructorArgs, apiKey: "test" }; + const chatModel = new this.Cls(params); + expect(chatModel).toBeDefined(); + } + + testChatModelInitStreaming() { + const params = { ...this.constructorArgs, streaming: true }; + const chatModel = new this.Cls(params); + expect(chatModel).toBeDefined(); + } + + testChatModelWithBindTools() { + if (!this.chatModelHasToolCalling) { + return; + } + const chatModel = new this.Cls(this.constructorArgs); + expect(chatModel.bindTools?.([new PersonTool()])).toBeDefined(); + } + + testChatModelWithStructuredOutput() { + if (!this.chatModelHasStructuredOutput) { + return; + } + const chatModel = new this.Cls(this.constructorArgs); + expect((chatModel as any).withStructuredOutput?.(person)).toBeDefined(); + } + + testStandardParams() { + const expectedParams = this.expectedLsParams(); + const chatModel = new this.Cls(this.constructorArgs); + + const lsParams = chatModel.getLsParams({} as any); + expect(lsParams).toBeDefined(); + expect(Object.keys(lsParams).sort()).toEqual( + Object.keys(expectedParams).sort() + ); + } + + /** + * Run all unit tests for the chat model. + * Each test is wrapped in a try/catch block to prevent the entire test suite from failing. + * If a test fails, the error is logged to the console, and the test suite continues. + * @returns {boolean} + */ + runTests(): boolean { + let allTestsPassed = true; + try { + this.testChatModelInit(); + } catch (e: any) { + allTestsPassed = false; + console.error("testChatModelInit failed", e); + } + + try { + this.testChatModelInitApiKey(); + } catch (e: any) { + allTestsPassed = false; + console.error("testChatModelInitApiKey failed", e); + } + + try { + this.testChatModelInitStreaming(); + } catch (e: any) { + allTestsPassed = false; + console.error("testChatModelInitStreaming failed", e); + } + + try { + this.testChatModelWithBindTools(); + } catch (e: any) { + allTestsPassed = false; + console.error("testChatModelWithBindTools failed", e); + } + + try { + this.testChatModelWithStructuredOutput(); + } catch (e: any) { + allTestsPassed = false; + console.error("testChatModelWithStructuredOutput failed", e); + } + + try { + this.testStandardParams(); + } catch (e: any) { + allTestsPassed = false; + console.error("testStandardParams failed", e); + } + + return allTestsPassed; + } +} diff --git a/libs/langchain-standard-tests/tsconfig.cjs.json b/libs/langchain-standard-tests/tsconfig.cjs.json new file mode 100644 index 000000000000..510d791e34ab --- /dev/null +++ b/libs/langchain-standard-tests/tsconfig.cjs.json @@ -0,0 +1,7 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "declaration": false + }, + "exclude": ["node_modules", "dist", "docs", "**/tests"] +} diff --git a/libs/langchain-standard-tests/tsconfig.json b/libs/langchain-standard-tests/tsconfig.json new file mode 100644 index 000000000000..446862d6ffc6 --- /dev/null +++ b/libs/langchain-standard-tests/tsconfig.json @@ -0,0 +1,23 @@ +{ + "extends": "@tsconfig/recommended", + "compilerOptions": { + "outDir": "../dist", + "rootDir": "./src", + "target": "ES2021", + "lib": ["ES2021", "ES2022.Object", "DOM"], + "module": "nodenext", + "moduleResolution": "nodenext", + "esModuleInterop": true, + "declaration": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "useDefineForClassFields": true, + "strictPropertyInitialization": false, + "allowJs": true, + "strict": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "docs"] +} diff --git a/libs/langchain-standard-tests/turbo.json b/libs/langchain-standard-tests/turbo.json new file mode 100644 index 000000000000..d024cee15c81 --- /dev/null +++ b/libs/langchain-standard-tests/turbo.json @@ -0,0 +1,11 @@ +{ + "extends": ["//"], + "pipeline": { + "build": { + "outputs": ["**/dist/**"] + }, + "build:internal": { + "dependsOn": ["^build:internal"] + } + } +} diff --git a/package.json b/package.json index 82c1d5abb96b..3aaa400da2d4 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,9 @@ "test:int:deps:down": "docker compose -f test-int-deps-docker-compose.yml down", "test:ranges:docker": "docker compose -f dependency_range_tests/docker-compose.yml up --force-recreate", "test:exports:docker": "docker compose -f environment_tests/docker-compose.yml up --force-recreate", + "test:standard:unit": "turbo test:standard:unit", + "test:standard:int": "turbo test:standard:int", + "test:standard": "yarn test:standard:unit && yarn test:standard:int", "example": "yarn workspace examples start", "precommit": "turbo precommit", "docs": "yarn workspace core_docs start", diff --git a/turbo.json b/turbo.json index 0f82b8628676..f69adb6cf3dc 100644 --- a/turbo.json +++ b/turbo.json @@ -32,6 +32,18 @@ "test:integration": { "dependsOn": ["^build", "build"] }, + "test:standard:unit": { + "outputs": [], + "dependsOn": ["^build"] + }, + "test:standard:int": { + "outputs": [], + "dependsOn": ["^build"] + }, + "test:standard": { + "outputs": [], + "dependsOn": ["^build"] + }, "clean": { "dependsOn": ["^clean"] }, diff --git a/yarn.lock b/yarn.lock index 3ab88496cd0e..6237d5511654 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8931,6 +8931,7 @@ __metadata: "@langchain/community": "workspace:*" "@langchain/core": ">=0.2.5 <0.3.0" "@langchain/scripts": ~0.0.14 + "@langchain/standard-tests": "workspace:*" "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 dpdm: ^3.12.0 @@ -8963,6 +8964,7 @@ __metadata: "@jest/globals": ^29.5.0 "@langchain/core": ">0.1.0 <0.3.0" "@langchain/scripts": ~0.0.14 + "@langchain/standard-tests": "workspace:*" "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 dpdm: ^3.12.0 @@ -8994,6 +8996,7 @@ __metadata: "@jest/globals": ^29.5.0 "@langchain/core": ">0.1.0 <0.3.0" "@langchain/scripts": ~0.0.14 + "@langchain/standard-tests": "workspace:*" "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 "@tsconfig/recommended": ^1.0.3 @@ -9036,6 +9039,7 @@ __metadata: "@jest/globals": ^29.5.0 "@langchain/core": ">0.1.58 <0.3.0" "@langchain/scripts": ~0.0.14 + "@langchain/standard-tests": "workspace:*" "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 "@tsconfig/recommended": ^1.0.3 @@ -9097,9 +9101,10 @@ __metadata: "@langchain/core": ~0.2.0 "@langchain/openai": ~0.1.0 "@langchain/scripts": ~0.0.14 + "@langchain/standard-tests": "workspace:*" "@layerup/layerup-security": ^1.5.12 "@mendable/firecrawl-js": ^0.0.13 - "@mlc-ai/web-llm": ^0.2.35 + "@mlc-ai/web-llm": ^0.2.40 "@mozilla/readability": ^0.4.4 "@neondatabase/serverless": ^0.9.1 "@notionhq/client": ^2.2.10 @@ -9266,7 +9271,7 @@ __metadata: "@huggingface/inference": ^2.6.4 "@layerup/layerup-security": ^1.5.12 "@mendable/firecrawl-js": ^0.0.13 - "@mlc-ai/web-llm": ^0.2.35 + "@mlc-ai/web-llm": ^0.2.40 "@mozilla/readability": "*" "@neondatabase/serverless": "*" "@notionhq/client": ^2.2.10 @@ -9715,6 +9720,7 @@ __metadata: "@langchain/core": ">0.1.56 <0.3.0" "@langchain/google-common": ~0.0.17 "@langchain/scripts": ~0.0.14 + "@langchain/standard-tests": "workspace:*" "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 "@tsconfig/recommended": ^1.0.3 @@ -9748,6 +9754,7 @@ __metadata: "@jest/globals": ^29.5.0 "@langchain/core": ">0.1.5 <0.3.0" "@langchain/scripts": ~0.0.14 + "@langchain/standard-tests": "workspace:*" "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 "@tsconfig/recommended": ^1.0.3 @@ -9877,6 +9884,7 @@ __metadata: "@langchain/core": ">0.1.56 <0.3.0" "@langchain/openai": "workspace:^" "@langchain/scripts": ~0.0.14 + "@langchain/standard-tests": "workspace:*" "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 "@tsconfig/recommended": ^1.0.3 @@ -9927,6 +9935,7 @@ __metadata: "@jest/globals": ^29.5.0 "@langchain/core": ">0.1.56 <0.3.0" "@langchain/scripts": ~0.0.14 + "@langchain/standard-tests": "workspace:*" "@mistralai/mistralai": ^0.4.0 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 @@ -10043,6 +10052,7 @@ __metadata: "@jest/globals": ^29.5.0 "@langchain/core": ">=0.2.5 <0.3.0" "@langchain/scripts": ~0.0.14 + "@langchain/standard-tests": "workspace:*" "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 dpdm: ^3.12.0 @@ -10206,6 +10216,37 @@ __metadata: languageName: unknown linkType: soft +"@langchain/standard-tests@workspace:*, @langchain/standard-tests@workspace:libs/langchain-standard-tests": + version: 0.0.0-use.local + resolution: "@langchain/standard-tests@workspace:libs/langchain-standard-tests" + dependencies: + "@jest/globals": ^29.5.0 + "@langchain/core": "workspace:*" + "@langchain/scripts": "workspace:*" + "@swc/core": ^1.3.90 + "@swc/jest": ^0.2.29 + "@tsconfig/recommended": ^1.0.3 + "@typescript-eslint/eslint-plugin": ^6.12.0 + "@typescript-eslint/parser": ^6.12.0 + dotenv: ^16.3.1 + dpdm: ^3.12.0 + eslint: ^8.33.0 + eslint-config-airbnb-base: ^15.0.0 + eslint-config-prettier: ^8.6.0 + eslint-plugin-import: ^2.27.5 + eslint-plugin-no-instanceof: ^1.0.1 + eslint-plugin-prettier: ^4.2.1 + jest: ^29.5.0 + jest-environment-node: ^29.6.4 + prettier: ^2.8.3 + release-it: ^15.10.1 + rollup: ^4.5.2 + ts-jest: ^29.1.0 + typescript: ^5.4.5 + zod: ^3.22.4 + languageName: unknown + linkType: soft + "@langchain/textsplitters@workspace:*, @langchain/textsplitters@workspace:libs/langchain-textsplitters, @langchain/textsplitters@~0.0.0": version: 0.0.0-use.local resolution: "@langchain/textsplitters@workspace:libs/langchain-textsplitters" @@ -10445,10 +10486,12 @@ __metadata: languageName: node linkType: hard -"@mlc-ai/web-llm@npm:^0.2.35": - version: 0.2.35 - resolution: "@mlc-ai/web-llm@npm:0.2.35" - checksum: 03c1d1847340f88474e1eeed7a91cc09e29299a1216e378385ffe5479c203d39a8656d98c9187864322453a91f046b874d7073662ab04033527079d9bb29bee3 +"@mlc-ai/web-llm@npm:^0.2.40": + version: 0.2.40 + resolution: "@mlc-ai/web-llm@npm:0.2.40" + dependencies: + loglevel: ^1.9.1 + checksum: 44d46178f7b7f899893ee8096fd4188b8c343589a10428c52f87b1b7e708f7a94b2b6315c8a6f8075f14d6d92aebfd8afc7f6d049a2ef60f8b8dc950b98a82e2 languageName: node linkType: hard @@ -28431,6 +28474,13 @@ __metadata: languageName: node linkType: hard +"loglevel@npm:^1.9.1": + version: 1.9.1 + resolution: "loglevel@npm:1.9.1" + checksum: e1c8586108c4d566122e91f8a79c8df728920e3a714875affa5120566761a24077ec8ec9e5fc388b022e39fc411ec6e090cde1b5775871241b045139771eeb06 + languageName: node + linkType: hard + "long@npm:*, long@npm:^5.2.1, long@npm:~5.2.3": version: 5.2.3 resolution: "long@npm:5.2.3"