From e834086e24538a73314e105321eac257421f6445 Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Tue, 4 Jun 2024 16:58:23 -0700 Subject: [PATCH] partners[minor]: Add standard chat model tests to partner packages (#5660) * partners[minor]: Add standard chat model tests to partner packages * google genai * yarn * groq * groq nit and mistral * add to azure in chat openai * chore: lint files * drop azure openai pkg * add generic constructor args to standard tests pkg * implement cloudflare standard tests * implement cohere standard tests * google genai package standard tests * groq * allow for custom function call ids, fix mistral * azure tests * chore: lint files * update standard tests gh action to run all pkgs * chore: lint files * revert workflow file rename * fix workflow job naming issue * add anthropic, fix api keys * cache deps? * fix build * update standard tests * cr * fix * remove dep on job which doesnt exist * cr * cr --- .github/workflows/standard-tests.yml | 57 +++++++++++++- libs/langchain-azure-openai/package.json | 17 ++--- libs/langchain-cloudflare/package.json | 13 ++-- libs/langchain-cloudflare/src/chat_models.ts | 10 +++ .../tests/chat_models.standard.int.test.ts | 53 +++++++++++++ .../src/tests/chat_models.standard.test.ts | 50 ++++++++++++ libs/langchain-cohere/package.json | 13 ++-- libs/langchain-cohere/src/chat_models.ts | 2 +- .../tests/chat_models.standard.int.test.ts | 47 ++++++++++++ .../src/tests/chat_models.standard.test.ts | 39 ++++++++++ libs/langchain-google-gauth/package.json | 22 +++--- libs/langchain-google-genai/package.json | 22 +++--- .../tests/chat_models.standard.int.test.ts | 76 +++++++++++++++++++ .../src/tests/chat_models.standard.test.ts | 42 ++++++++++ libs/langchain-groq/package.json | 14 ++-- libs/langchain-groq/src/chat_models.ts | 26 ++++++- .../tests/chat_models.standard.int.test.ts | 57 ++++++++++++++ .../src/tests/chat_models.standard.test.ts | 39 ++++++++++ libs/langchain-mistralai/package.json | 14 ++-- libs/langchain-mistralai/src/chat_models.ts | 2 + .../tests/chat_models.standard.int.test.ts | 49 ++++++++++++ .../src/tests/chat_models.standard.test.ts | 53 +++++++++++++ .../azure/chat_models.standard.int.test.ts | 45 +++++++++++ .../tests/azure/chat_models.standard.test.ts | 37 +++++++++ libs/langchain-standard-tests/src/base.ts | 60 ++++++++++++--- .../src/integration_tests/chat_models.ts | 33 ++++++-- .../src/unit_tests/chat_models.ts | 36 ++++++--- yarn.lock | 7 ++ 28 files changed, 829 insertions(+), 106 deletions(-) create mode 100644 libs/langchain-cloudflare/src/tests/chat_models.standard.int.test.ts create mode 100644 libs/langchain-cloudflare/src/tests/chat_models.standard.test.ts create mode 100644 libs/langchain-cohere/src/tests/chat_models.standard.int.test.ts create mode 100644 libs/langchain-cohere/src/tests/chat_models.standard.test.ts create mode 100644 libs/langchain-google-genai/src/tests/chat_models.standard.int.test.ts create mode 100644 libs/langchain-google-genai/src/tests/chat_models.standard.test.ts create mode 100644 libs/langchain-groq/src/tests/chat_models.standard.int.test.ts create mode 100644 libs/langchain-groq/src/tests/chat_models.standard.test.ts create mode 100644 libs/langchain-mistralai/src/tests/chat_models.standard.int.test.ts create mode 100644 libs/langchain-mistralai/src/tests/chat_models.standard.test.ts create mode 100644 libs/langchain-openai/src/tests/azure/chat_models.standard.int.test.ts create mode 100644 libs/langchain-openai/src/tests/azure/chat_models.standard.test.ts diff --git a/.github/workflows/standard-tests.yml b/.github/workflows/standard-tests.yml index 5411831078b7..5be8ef362d65 100644 --- a/.github/workflows/standard-tests.yml +++ b/.github/workflows/standard-tests.yml @@ -3,11 +3,14 @@ name: Standard Tests (Integration) on: workflow_dispatch: schedule: - - cron: '0 13 * * *' + - cron: '0 13 * * *' jobs: - openai: + standard-tests: runs-on: ubuntu-latest + strategy: + matrix: + package: [anthropic, cohere, google-genai, groq, mistralai] steps: - uses: actions/checkout@v4 - name: Use Node.js 18.x @@ -17,7 +20,53 @@ jobs: cache: "yarn" - name: Install dependencies run: yarn install --immutable --mode=skip-build - - name: Run standard tests (integration) - run: yarn test:standard:int --filter=@langchain/openai + - name: Run standard tests (integration) for ${{ matrix.package }} + run: yarn test:standard:int --filter=@langchain/${{ matrix.package }} + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} + MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} + + # The `@langchain/openai` package contains standard tests for ChatOpenAI and AzureChatOpenAI + # We want to run these separately, so we need to pass the exact path for each test, which means + # we need separate jobs for each test. + standard-tests-openai: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Use Node.js 18.x + uses: actions/setup-node@v3 + with: + node-version: 18.x + cache: "yarn" + - name: Install dependencies + run: yarn install --immutable --mode=skip-build + - name: Build `@langchain/openai` + run: yarn build --filter=@langchain/openai + - name: Run standard tests (integration) for ChatOpenAI + run: yarn workspace @langchain/openai test:single src/tests/chat_models.standard.int.test.ts env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + + standard-tests-azure-openai: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Use Node.js 18.x + uses: actions/setup-node@v3 + with: + node-version: 18.x + cache: "yarn" + - name: Install dependencies + run: yarn install --immutable --mode=skip-build + - name: Build `@langchain/openai` + run: yarn build --filter=@langchain/openai + - name: Run standard tests (integration) for `@langchain/openai` AzureChatOpenAI + run: yarn workspace @langchain/openai test:single src/tests/azure/chat_models.standard.int.test.ts + env: + AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} + AZURE_OPENAI_API_DEPLOYMENT_NAME: "chat" + AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION }} + AZURE_OPENAI_BASE_PATH: ${{ secrets.AZURE_OPENAI_BASE_PATH }} diff --git a/libs/langchain-azure-openai/package.json b/libs/langchain-azure-openai/package.json index 17c6c15a5f7c..e7f498c05c18 100644 --- a/libs/langchain-azure-openai/package.json +++ b/libs/langchain-azure-openai/package.json @@ -15,26 +15,18 @@ "scripts": { "build": "yarn turbo:command build:internal --filter=@langchain/azure-openai", "build:internal": "yarn lc-build:v2 --create-entrypoints --pre --tree-shaking", - "build:deps": "yarn run turbo:command build --filter=@langchain/core", - "build:esm": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist/", - "build:cjs": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist-cjs/ -p tsconfig.cjs.json && yarn move-cjs-to-dist && rimraf dist-cjs", - "build:watch": "yarn create-entrypoints && tsc --outDir dist/ --watch", - "build:scripts": "yarn create-entrypoints && yarn check-tree-shaking", "lint:eslint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/", "lint:dpdm": "dpdm --exit-code circular:1 --no-warning --no-tree src/*.ts src/**/*.ts", "lint": "yarn lint:eslint && yarn lint:dpdm", "lint:fix": "yarn lint:eslint --fix && yarn lint:dpdm", "clean": "rm -rf .turbo dist/", "prepack": "yarn build", - "test": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%", - "test:watch": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", - "test:single": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", + "test": "NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%", + "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", + "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", "test:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", "format": "prettier --config .prettierrc --write \"src\"", - "format:check": "prettier --config .prettierrc --check \"src\"", - "move-cjs-to-dist": "yarn lc-build --config ./langchain.config.js --move-cjs-dist", - "create-entrypoints": "yarn lc-build --config ./langchain.config.js --create-entrypoints", - "check-tree-shaking": "yarn lc-build --config ./langchain.config.js --tree-shaking" + "format:check": "prettier --config .prettierrc --check \"src\"" }, "author": "LangChain", "license": "MIT", @@ -50,6 +42,7 @@ "@azure/identity": "^4.0.1", "@jest/globals": "^29.5.0", "@langchain/scripts": "~0.0.14", + "@langchain/standard-tests": "workspace:*", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "dpdm": "^3.12.0", diff --git a/libs/langchain-cloudflare/package.json b/libs/langchain-cloudflare/package.json index f6141d965bb5..8b5e6d9afefd 100644 --- a/libs/langchain-cloudflare/package.json +++ b/libs/langchain-cloudflare/package.json @@ -16,10 +16,6 @@ "scripts": { "build": "yarn turbo:command build:internal --filter=@langchain/cloudflare", "build:internal": "yarn lc-build:v2 --create-entrypoints --pre --tree-shaking", - "build:esm": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist/ && rm -rf dist/tests dist/**/tests", - "build:cjs": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist-cjs/ -p tsconfig.cjs.json && yarn move-cjs-to-dist && rm -rf dist-cjs", - "build:watch": "yarn create-entrypoints && tsc --outDir dist/ --watch", - "build:scripts": "yarn create-entrypoints && yarn check-tree-shaking", "lint:eslint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/", "lint:dpdm": "dpdm --exit-code circular:1 --no-warning --no-tree src/*.ts src/**/*.ts", "lint": "yarn lint:eslint && yarn lint:dpdm", @@ -30,11 +26,11 @@ "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", "test:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:unit": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard": "yarn test:standard:unit && yarn test:standard:int", "format": "prettier --config .prettierrc --write \"src\"", - "format:check": "prettier --config .prettierrc --check \"src\"", - "move-cjs-to-dist": "yarn lc-build --config ./langchain.config.js --move-cjs-dist", - "create-entrypoints": "yarn lc-build --config ./langchain.config.js --create-entrypoints", - "check-tree-shaking": "yarn lc-build --config ./langchain.config.js --tree-shaking" + "format:check": "prettier --config .prettierrc --check \"src\"" }, "author": "LangChain", "license": "MIT", @@ -47,6 +43,7 @@ "@cloudflare/workers-types": "^4.20231218.0", "@jest/globals": "^29.5.0", "@langchain/scripts": "~0.0.14", + "@langchain/standard-tests": "workspace:*", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "@tsconfig/recommended": "^1.0.3", diff --git a/libs/langchain-cloudflare/src/chat_models.ts b/libs/langchain-cloudflare/src/chat_models.ts index af4576d18654..bdae5020d6ba 100644 --- a/libs/langchain-cloudflare/src/chat_models.ts +++ b/libs/langchain-cloudflare/src/chat_models.ts @@ -1,4 +1,5 @@ import { + LangSmithParams, SimpleChatModel, type BaseChatModelParams, } from "@langchain/core/language_models/chat_models"; @@ -81,6 +82,15 @@ export class ChatCloudflareWorkersAI } } + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { + return { + ls_provider: "openai", + ls_model_name: this.model, + ls_model_type: "chat", + ls_stop: options.stop, + }; + } + get lc_secrets(): { [key: string]: string } | undefined { return { cloudflareApiToken: "CLOUDFLARE_API_TOKEN", diff --git a/libs/langchain-cloudflare/src/tests/chat_models.standard.int.test.ts b/libs/langchain-cloudflare/src/tests/chat_models.standard.int.test.ts new file mode 100644 index 000000000000..c49d3b7d009d --- /dev/null +++ b/libs/langchain-cloudflare/src/tests/chat_models.standard.int.test.ts @@ -0,0 +1,53 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelIntegrationTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { + ChatCloudflareWorkersAI, + ChatCloudflareWorkersAICallOptions, +} from "../chat_models.js"; + +class ChatCloudflareWorkersAIStandardIntegrationTests extends ChatModelIntegrationTests< + ChatCloudflareWorkersAICallOptions, + AIMessageChunk +> { + constructor() { + if ( + !process.env.CLOUDFLARE_ACCOUNT_ID || + !process.env.CLOUDFLARE_API_TOKEN + ) { + throw new Error( + "Skipping Cloudflare Workers AI integration tests because CLOUDFLARE_ACCOUNT_ID or CLOUDFLARE_API_TOKEN is not set" + ); + } + super({ + Cls: ChatCloudflareWorkersAI, + chatModelHasToolCalling: false, + chatModelHasStructuredOutput: false, + constructorArgs: {}, + }); + } + + async testUsageMetadataStreaming() { + this.skipTestMessage( + "testUsageMetadataStreaming", + "ChatCloudflareWorkersAI", + "Streaming tokens is not currently supported." + ); + } + + async testUsageMetadata() { + this.skipTestMessage( + "testUsageMetadata", + "ChatCloudflareWorkersAI", + "Usage metadata tokens is not currently supported." + ); + } +} + +const testClass = new ChatCloudflareWorkersAIStandardIntegrationTests(); + +test("ChatCloudflareWorkersAIStandardIntegrationTests", async () => { + const testResults = await testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-cloudflare/src/tests/chat_models.standard.test.ts b/libs/langchain-cloudflare/src/tests/chat_models.standard.test.ts new file mode 100644 index 000000000000..1faa95c782e2 --- /dev/null +++ b/libs/langchain-cloudflare/src/tests/chat_models.standard.test.ts @@ -0,0 +1,50 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelUnitTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { LangSmithParams } from "@langchain/core/language_models/chat_models"; +import { + ChatCloudflareWorkersAI, + ChatCloudflareWorkersAICallOptions, +} from "../chat_models.js"; + +class ChatCloudflareWorkersAIStandardUnitTests extends ChatModelUnitTests< + ChatCloudflareWorkersAICallOptions, + AIMessageChunk +> { + constructor() { + super({ + Cls: ChatCloudflareWorkersAI, + chatModelHasToolCalling: false, + chatModelHasStructuredOutput: false, + constructorArgs: {}, + }); + } + + testChatModelInitApiKey() { + this.skipTestMessage( + "testChatModelInitApiKey", + "ChatCloudflareWorkersAI", + this.multipleApiKeysRequiredMessage + ); + } + + expectedLsParams(): Partial { + console.warn( + "Overriding testStandardParams. ChatCloudflareWorkersAI does not support temperature or max tokens." + ); + return { + ls_provider: "string", + ls_model_name: "string", + ls_model_type: "chat", + ls_stop: ["Array"], + }; + } +} + +const testClass = new ChatCloudflareWorkersAIStandardUnitTests(); + +test("ChatCloudflareWorkersAIStandardUnitTests", () => { + const testResults = testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-cohere/package.json b/libs/langchain-cohere/package.json index 220b7f2f635f..6aac0d2cddd5 100644 --- a/libs/langchain-cohere/package.json +++ b/libs/langchain-cohere/package.json @@ -16,10 +16,6 @@ "scripts": { "build": "yarn turbo:command build:internal --filter=@langchain/cohere", "build:internal": "yarn lc-build:v2 --create-entrypoints --pre --tree-shaking", - "build:esm": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist/ && rm -rf dist/tests dist/**/tests", - "build:cjs": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist-cjs/ -p tsconfig.cjs.json && yarn move-cjs-to-dist && rm -rf dist-cjs", - "build:watch": "yarn create-entrypoints && tsc --outDir dist/ --watch", - "build:scripts": "yarn create-entrypoints && yarn check-tree-shaking", "lint:eslint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/", "lint:dpdm": "dpdm --exit-code circular:1 --no-warning --no-tree src/*.ts src/**/*.ts", "lint": "yarn lint:eslint && yarn lint:dpdm", @@ -30,11 +26,11 @@ "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", "test:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:unit": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard": "yarn test:standard:unit && yarn test:standard:int", "format": "prettier --config .prettierrc --write \"src\"", - "format:check": "prettier --config .prettierrc --check \"src\"", - "move-cjs-to-dist": "yarn lc-build --config ./langchain.config.js --move-cjs-dist", - "create-entrypoints": "yarn lc-build --config ./langchain.config.js --create-entrypoints", - "check-tree-shaking": "yarn lc-build --config ./langchain.config.js --tree-shaking" + "format:check": "prettier --config .prettierrc --check \"src\"" }, "author": "LangChain", "license": "MIT", @@ -45,6 +41,7 @@ "devDependencies": { "@jest/globals": "^29.5.0", "@langchain/scripts": "~0.0.14", + "@langchain/standard-tests": "workspace:*", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "@tsconfig/recommended": "^1.0.3", diff --git a/libs/langchain-cohere/src/chat_models.ts b/libs/langchain-cohere/src/chat_models.ts index d88145ae8ba1..2179ec0bbc5d 100644 --- a/libs/langchain-cohere/src/chat_models.ts +++ b/libs/langchain-cohere/src/chat_models.ts @@ -55,7 +55,7 @@ interface TokenUsage { totalTokens?: number; } -interface CohereChatCallOptions +export interface CohereChatCallOptions extends BaseLanguageModelCallOptions, Partial>, Partial> {} diff --git a/libs/langchain-cohere/src/tests/chat_models.standard.int.test.ts b/libs/langchain-cohere/src/tests/chat_models.standard.int.test.ts new file mode 100644 index 000000000000..813aa00da412 --- /dev/null +++ b/libs/langchain-cohere/src/tests/chat_models.standard.int.test.ts @@ -0,0 +1,47 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelIntegrationTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { ChatCohere, CohereChatCallOptions } from "../chat_models.js"; + +class ChatCohereStandardIntegrationTests extends ChatModelIntegrationTests< + CohereChatCallOptions, + AIMessageChunk +> { + constructor() { + if (!process.env.COHERE_API_KEY) { + throw new Error( + "Can not run Cohere integration tests because COHERE_API_KEY is not set" + ); + } + super({ + Cls: ChatCohere, + chatModelHasToolCalling: false, + chatModelHasStructuredOutput: false, + constructorArgs: {}, + }); + } + + async testUsageMetadataStreaming() { + this.skipTestMessage( + "testUsageMetadataStreaming", + "ChatCohere", + "Streaming tokens is not currently supported." + ); + } + + async testUsageMetadata() { + this.skipTestMessage( + "testUsageMetadata", + "ChatCohere", + "Usage metadata tokens is not currently supported." + ); + } +} + +const testClass = new ChatCohereStandardIntegrationTests(); + +test("ChatCohereStandardIntegrationTests", async () => { + const testResults = await testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-cohere/src/tests/chat_models.standard.test.ts b/libs/langchain-cohere/src/tests/chat_models.standard.test.ts new file mode 100644 index 000000000000..dbfc2813ae83 --- /dev/null +++ b/libs/langchain-cohere/src/tests/chat_models.standard.test.ts @@ -0,0 +1,39 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelUnitTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { ChatCohere, CohereChatCallOptions } from "../chat_models.js"; + +class ChatCohereStandardUnitTests extends ChatModelUnitTests< + CohereChatCallOptions, + AIMessageChunk +> { + constructor() { + super({ + Cls: ChatCohere, + chatModelHasToolCalling: false, + chatModelHasStructuredOutput: false, + constructorArgs: {}, + }); + // This must be set so method like `.bindTools` or `.withStructuredOutput` + // which we call after instantiating the model will work. + // (constructor will throw if API key is not set) + process.env.COHERE_API_KEY = "test"; + } + + testChatModelInitApiKey() { + // Unset the API key env var here so this test can properly check + // the API key class arg. + process.env.COHERE_API_KEY = ""; + super.testChatModelInitApiKey(); + // Re-set the API key env var here so other tests can run properly. + process.env.COHERE_API_KEY = "test"; + } +} + +const testClass = new ChatCohereStandardUnitTests(); + +test("ChatCohereStandardUnitTests", () => { + const testResults = testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-google-gauth/package.json b/libs/langchain-google-gauth/package.json index 57b9db09a991..341d5c22d750 100644 --- a/libs/langchain-google-gauth/package.json +++ b/libs/langchain-google-gauth/package.json @@ -16,26 +16,21 @@ "scripts": { "build": "yarn turbo:command build:internal --filter=@langchain/google-gauth", "build:internal": "yarn lc-build:v2 --create-entrypoints --pre --tree-shaking", - "build:deps": "yarn run turbo:command build --filter=@langchain/google-common", - "build:esm": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist/ && rm -rf dist/tests dist/**/tests", - "build:cjs": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist-cjs/ -p tsconfig.cjs.json && yarn move-cjs-to-dist && rm -rf dist-cjs", - "build:watch": "yarn create-entrypoints && tsc --outDir dist/ --watch", - "build:scripts": "yarn create-entrypoints && yarn check-tree-shaking", "lint:eslint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/", "lint:dpdm": "dpdm --exit-code circular:1 --no-warning --no-tree src/*.ts src/**/*.ts", "lint": "yarn lint:eslint && yarn lint:dpdm", "lint:fix": "yarn lint:eslint --fix && yarn lint:dpdm", "clean": "rm -rf .turbo dist/", "prepack": "yarn build", - "test": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%", - "test:watch": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", - "test:single": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", - "test:integration": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test": "NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%", + "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", + "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", + "test:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:unit": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard": "yarn test:standard:unit && yarn test:standard:int", "format": "prettier --config .prettierrc --write \"src\"", - "format:check": "prettier --config .prettierrc --check \"src\"", - "move-cjs-to-dist": "yarn lc-build --config ./langchain.config.js --move-cjs-dist", - "create-entrypoints": "yarn lc-build --config ./langchain.config.js --create-entrypoints", - "check-tree-shaking": "yarn lc-build --config ./langchain.config.js --tree-shaking" + "format:check": "prettier --config .prettierrc --check \"src\"" }, "author": "LangChain", "license": "MIT", @@ -47,6 +42,7 @@ "devDependencies": { "@jest/globals": "^29.5.0", "@langchain/scripts": "~0.0.14", + "@langchain/standard-tests": "workspace:*", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "@tsconfig/recommended": "^1.0.3", diff --git a/libs/langchain-google-genai/package.json b/libs/langchain-google-genai/package.json index d3443dbfbdfa..b5890cefd885 100644 --- a/libs/langchain-google-genai/package.json +++ b/libs/langchain-google-genai/package.json @@ -16,26 +16,21 @@ "scripts": { "build": "yarn turbo:command build:internal --filter=@langchain/google-genai", "build:internal": "yarn lc-build:v2 --create-entrypoints --pre --tree-shaking", - "build:deps": "yarn run turbo:command build --filter=@langchain/core", - "build:esm": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist/ && rm -rf dist/tests dist/**/tests", - "build:cjs": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist-cjs/ -p tsconfig.cjs.json && yarn move-cjs-to-dist && rm -rf dist-cjs", - "build:watch": "yarn create-entrypoints && tsc --outDir dist/ --watch", - "build:scripts": "yarn create-entrypoints && yarn check-tree-shaking", "lint:eslint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/", "lint:dpdm": "dpdm --exit-code circular:1 --no-warning --no-tree src/*.ts src/**/*.ts", "lint": "yarn lint:eslint && yarn lint:dpdm", "lint:fix": "yarn lint:eslint --fix && yarn lint:dpdm", "clean": "rm -rf .turbo dist/", "prepack": "yarn build", - "test": "yarn build:deps && NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%", - "test:watch": "yarn build:deps && NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", - "test:single": "yarn build:deps && NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", - "test:int": "yarn build:deps && NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test": "NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%", + "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", + "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", + "test:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:unit": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard": "yarn test:standard:unit && yarn test:standard:int", "format": "prettier --config .prettierrc --write \"src\"", - "format:check": "prettier --config .prettierrc --check \"src\"", - "move-cjs-to-dist": "yarn lc-build --config ./langchain.config.js --move-cjs-dist", - "create-entrypoints": "yarn lc-build --config ./langchain.config.js --create-entrypoints", - "check-tree-shaking": "yarn lc-build --config ./langchain.config.js --tree-shaking" + "format:check": "prettier --config .prettierrc --check \"src\"" }, "author": "LangChain", "license": "MIT", @@ -47,6 +42,7 @@ "devDependencies": { "@jest/globals": "^29.5.0", "@langchain/scripts": "~0.0.14", + "@langchain/standard-tests": "workspace:*", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "@tsconfig/recommended": "^1.0.3", diff --git a/libs/langchain-google-genai/src/tests/chat_models.standard.int.test.ts b/libs/langchain-google-genai/src/tests/chat_models.standard.int.test.ts new file mode 100644 index 000000000000..4f9909358165 --- /dev/null +++ b/libs/langchain-google-genai/src/tests/chat_models.standard.int.test.ts @@ -0,0 +1,76 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelIntegrationTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { + ChatGoogleGenerativeAI, + GoogleGenerativeAIChatCallOptions, +} from "../chat_models.js"; + +class ChatGoogleGenerativeAIStandardIntegrationTests extends ChatModelIntegrationTests< + GoogleGenerativeAIChatCallOptions, + AIMessageChunk +> { + constructor() { + if (!process.env.GOOGLE_API_KEY) { + throw new Error( + "Can not run Google Generative AI integration tests because GOOGLE_API_KEY is set" + ); + } + super({ + Cls: ChatGoogleGenerativeAI, + chatModelHasToolCalling: true, + chatModelHasStructuredOutput: true, + constructorArgs: { + maxRetries: 1, + }, + }); + } + + async testUsageMetadataStreaming() { + this.skipTestMessage( + "testUsageMetadataStreaming", + "ChatGoogleGenerativeAI", + "Streaming tokens is not currently supported." + ); + } + + async testUsageMetadata() { + this.skipTestMessage( + "testUsageMetadata", + "ChatGoogleGenerativeAI", + "Usage metadata tokens is not currently supported." + ); + } + + async testToolMessageHistoriesStringContent() { + this.skipTestMessage( + "testToolMessageHistoriesStringContent", + "ChatGoogleGenerativeAI", + "Not implemented." + ); + } + + async testToolMessageHistoriesListContent() { + this.skipTestMessage( + "testToolMessageHistoriesListContent", + "ChatGoogleGenerativeAI", + "Not implemented." + ); + } + + async testStructuredFewShotExamples() { + this.skipTestMessage( + "testStructuredFewShotExamples", + "ChatGoogleGenerativeAI", + ".bindTools not implemented properly." + ); + } +} + +const testClass = new ChatGoogleGenerativeAIStandardIntegrationTests(); + +test("ChatGoogleGenerativeAIStandardIntegrationTests", async () => { + const testResults = await testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-google-genai/src/tests/chat_models.standard.test.ts b/libs/langchain-google-genai/src/tests/chat_models.standard.test.ts new file mode 100644 index 000000000000..a7fa2ef2a3ae --- /dev/null +++ b/libs/langchain-google-genai/src/tests/chat_models.standard.test.ts @@ -0,0 +1,42 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelUnitTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { + ChatGoogleGenerativeAI, + GoogleGenerativeAIChatCallOptions, +} from "../chat_models.js"; + +class ChatGoogleGenerativeAIStandardUnitTests extends ChatModelUnitTests< + GoogleGenerativeAIChatCallOptions, + AIMessageChunk +> { + constructor() { + super({ + Cls: ChatGoogleGenerativeAI, + chatModelHasToolCalling: true, + chatModelHasStructuredOutput: true, + constructorArgs: {}, + }); + // This must be set so method like `.bindTools` or `.withStructuredOutput` + // which we call after instantiating the model will work. + // (constructor will throw if API key is not set) + process.env.GOOGLE_API_KEY = "test"; + } + + testChatModelInitApiKey() { + // Unset the API key env var here so this test can properly check + // the API key class arg. + process.env.GOOGLE_API_KEY = ""; + super.testChatModelInitApiKey(); + // Re-set the API key env var here so other tests can run properly. + process.env.GOOGLE_API_KEY = "test"; + } +} + +const testClass = new ChatGoogleGenerativeAIStandardUnitTests(); + +test("ChatGoogleGenerativeAIStandardUnitTests", () => { + const testResults = testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-groq/package.json b/libs/langchain-groq/package.json index 38f5ab175c44..8672bb54dc31 100644 --- a/libs/langchain-groq/package.json +++ b/libs/langchain-groq/package.json @@ -16,11 +16,6 @@ "scripts": { "build": "yarn turbo:command build:internal --filter=@langchain/groq", "build:internal": "yarn lc-build:v2 --create-entrypoints --pre --tree-shaking", - "build:deps": "yarn run turbo:command build --filter=@langchain/core", - "build:esm": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist/ && rm -rf dist/tests dist/**/tests", - "build:cjs": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist-cjs/ -p tsconfig.cjs.json && yarn move-cjs-to-dist && rm -rf dist-cjs", - "build:watch": "yarn create-entrypoints && tsc --outDir dist/ --watch", - "build:scripts": "yarn create-entrypoints && yarn check-tree-shaking", "lint:eslint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/", "lint:dpdm": "dpdm --exit-code circular:1 --no-warning --no-tree src/*.ts src/**/*.ts", "lint": "yarn lint:eslint && yarn lint:dpdm", @@ -31,11 +26,11 @@ "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", "test:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:unit": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard": "yarn test:standard:unit && yarn test:standard:int", "format": "prettier --config .prettierrc --write \"src\"", - "format:check": "prettier --config .prettierrc --check \"src\"", - "move-cjs-to-dist": "yarn lc-build --config ./langchain.config.js --move-cjs-dist", - "create-entrypoints": "yarn lc-build --config ./langchain.config.js --create-entrypoints", - "check-tree-shaking": "yarn lc-build --config ./langchain.config.js --tree-shaking" + "format:check": "prettier --config .prettierrc --check \"src\"" }, "author": "LangChain", "license": "MIT", @@ -50,6 +45,7 @@ "@jest/globals": "^29.5.0", "@langchain/openai": "workspace:^", "@langchain/scripts": "~0.0.14", + "@langchain/standard-tests": "workspace:*", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "@tsconfig/recommended": "^1.0.3", diff --git a/libs/langchain-groq/src/chat_models.ts b/libs/langchain-groq/src/chat_models.ts index 7dce52d8436b..acc43c2850d9 100644 --- a/libs/langchain-groq/src/chat_models.ts +++ b/libs/langchain-groq/src/chat_models.ts @@ -5,6 +5,7 @@ import { CallbackManagerForLLMRun } from "@langchain/core/callbacks/manager"; import { BaseChatModel, BaseChatModelCallOptions, + LangSmithParams, type BaseChatModelParams, } from "@langchain/core/language_models/chat_models"; import { @@ -312,6 +313,18 @@ export class ChatGroq extends BaseChatModel< this.maxTokens = fields?.maxTokens; } + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { + const params = this.invocationParams(options); + return { + ls_provider: "groq", + ls_model_name: this.model, + ls_model_type: "chat", + ls_temperature: params.temperature, + ls_max_tokens: params.max_tokens, + ls_stop: options.stop, + }; + } + async completionWithRetry( request: ChatCompletionCreateParamsStreaming, options?: OpenAICoreRequestOptions @@ -411,13 +424,24 @@ export class ChatGroq extends BaseChatModel< headers: options?.headers, } ); + let role = ""; for await (const data of response) { const choice = data?.choices[0]; if (!choice) { continue; } + // The `role` field is populated in the first delta of the response + // but is not present in subsequent deltas. Extract it when available. + if (choice.delta?.role) { + role = choice.delta.role; + } const chunk = new ChatGenerationChunk({ - message: _convertDeltaToMessageChunk(choice.delta ?? {}), + message: _convertDeltaToMessageChunk( + { + ...choice.delta, + role, + } ?? {} + ), text: choice.delta.content ?? "", generationInfo: { finishReason: choice.finish_reason, diff --git a/libs/langchain-groq/src/tests/chat_models.standard.int.test.ts b/libs/langchain-groq/src/tests/chat_models.standard.int.test.ts new file mode 100644 index 000000000000..4eb55e6cd5c7 --- /dev/null +++ b/libs/langchain-groq/src/tests/chat_models.standard.int.test.ts @@ -0,0 +1,57 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelIntegrationTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { ChatGroq, ChatGroqCallOptions } from "../chat_models.js"; + +class ChatGroqStandardIntegrationTests extends ChatModelIntegrationTests< + ChatGroqCallOptions, + AIMessageChunk +> { + constructor() { + if (!process.env.GROQ_API_KEY) { + throw new Error( + "Can not run Groq integration tests because GROQ_API_KEY is not set" + ); + } + super({ + Cls: ChatGroq, + chatModelHasToolCalling: true, + chatModelHasStructuredOutput: true, + constructorArgs: { + model: "mixtral-8x7b-32768", + }, + }); + } + + async testUsageMetadataStreaming() { + this.skipTestMessage( + "testUsageMetadataStreaming", + "ChatGroq", + "Streaming tokens is not currently supported." + ); + } + + async testUsageMetadata() { + this.skipTestMessage( + "testUsageMetadata", + "ChatGroq", + "Usage metadata tokens is not currently supported." + ); + } + + async testToolMessageHistoriesListContent() { + this.skipTestMessage( + "testToolMessageHistoriesListContent", + "ChatGroq", + "Not properly implemented." + ); + } +} + +const testClass = new ChatGroqStandardIntegrationTests(); + +test("ChatGroqStandardIntegrationTests", async () => { + const testResults = await testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-groq/src/tests/chat_models.standard.test.ts b/libs/langchain-groq/src/tests/chat_models.standard.test.ts new file mode 100644 index 000000000000..8678bfc87ca2 --- /dev/null +++ b/libs/langchain-groq/src/tests/chat_models.standard.test.ts @@ -0,0 +1,39 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelUnitTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { ChatGroq, ChatGroqCallOptions } from "../chat_models.js"; + +class ChatGroqStandardUnitTests extends ChatModelUnitTests< + ChatGroqCallOptions, + AIMessageChunk +> { + constructor() { + super({ + Cls: ChatGroq, + chatModelHasToolCalling: true, + chatModelHasStructuredOutput: true, + constructorArgs: {}, + }); + // This must be set so method like `.bindTools` or `.withStructuredOutput` + // which we call after instantiating the model will work. + // (constructor will throw if API key is not set) + process.env.GROQ_API_KEY = "test"; + } + + testChatModelInitApiKey() { + // Unset the API key env var here so this test can properly check + // the API key class arg. + process.env.GROQ_API_KEY = ""; + super.testChatModelInitApiKey(); + // Re-set the API key env var here so other tests can run properly. + process.env.GROQ_API_KEY = "test"; + } +} + +const testClass = new ChatGroqStandardUnitTests(); + +test("ChatGroqStandardUnitTests", () => { + const testResults = testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-mistralai/package.json b/libs/langchain-mistralai/package.json index 6429d3437040..762bee5fe213 100644 --- a/libs/langchain-mistralai/package.json +++ b/libs/langchain-mistralai/package.json @@ -16,11 +16,6 @@ "scripts": { "build": "yarn turbo:command build:internal --filter=@langchain/mistralai", "build:internal": "yarn lc-build:v2 --create-entrypoints --pre --tree-shaking", - "build:deps": "yarn run turbo:command build --filter=@langchain/core", - "build:esm": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist/ && rm -rf dist/tests dist/**/tests", - "build:cjs": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist-cjs/ -p tsconfig.cjs.json && yarn move-cjs-to-dist && rm -rf dist-cjs", - "build:watch": "yarn create-entrypoints && tsc --outDir dist/ --watch", - "build:scripts": "yarn create-entrypoints && yarn check-tree-shaking", "lint:eslint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/", "lint:dpdm": "dpdm --exit-code circular:1 --no-warning --no-tree src/*.ts src/**/*.ts", "lint": "yarn lint:eslint && yarn lint:dpdm", @@ -31,11 +26,11 @@ "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", "test:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:unit": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.standard\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "test:standard": "yarn test:standard:unit && yarn test:standard:int", "format": "prettier --config .prettierrc --write \"src\"", - "format:check": "prettier --config .prettierrc --check \"src\"", - "move-cjs-to-dist": "yarn lc-build --config ./langchain.config.js --move-cjs-dist", - "create-entrypoints": "yarn lc-build --config ./langchain.config.js --create-entrypoints", - "check-tree-shaking": "yarn lc-build --config ./langchain.config.js --tree-shaking" + "format:check": "prettier --config .prettierrc --check \"src\"" }, "author": "LangChain", "license": "MIT", @@ -49,6 +44,7 @@ "devDependencies": { "@jest/globals": "^29.5.0", "@langchain/scripts": "~0.0.14", + "@langchain/standard-tests": "workspace:*", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "@tsconfig/recommended": "^1.0.3", diff --git a/libs/langchain-mistralai/src/chat_models.ts b/libs/langchain-mistralai/src/chat_models.ts index ab9d9bff360b..a86589bd8dfd 100644 --- a/libs/langchain-mistralai/src/chat_models.ts +++ b/libs/langchain-mistralai/src/chat_models.ts @@ -82,6 +82,8 @@ interface MistralAICallOptions tool_choice?: MistralAIToolChoice; } +export interface ChatMistralAICallOptions extends MistralAICallOptions {} + /** * Input to chat model class. */ diff --git a/libs/langchain-mistralai/src/tests/chat_models.standard.int.test.ts b/libs/langchain-mistralai/src/tests/chat_models.standard.int.test.ts new file mode 100644 index 000000000000..0d80e46fccdb --- /dev/null +++ b/libs/langchain-mistralai/src/tests/chat_models.standard.int.test.ts @@ -0,0 +1,49 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelIntegrationTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { ChatMistralAI, ChatMistralAICallOptions } from "../chat_models.js"; + +class ChatMistralAIStandardIntegrationTests extends ChatModelIntegrationTests< + ChatMistralAICallOptions, + AIMessageChunk +> { + constructor() { + if (!process.env.MISTRAL_API_KEY) { + throw new Error( + "Can not run Mistral AI integration tests because MISTRAL_API_KEY is not set" + ); + } + super({ + Cls: ChatMistralAI, + chatModelHasToolCalling: true, + chatModelHasStructuredOutput: true, + constructorArgs: {}, + // Mistral requires function call IDs to be a-z, A-Z, 0-9, with a length of 9. + functionId: "123456789", + }); + } + + async testUsageMetadataStreaming() { + this.skipTestMessage( + "testUsageMetadataStreaming", + "ChatMistralAI", + "Streaming tokens is not currently supported." + ); + } + + async testUsageMetadata() { + this.skipTestMessage( + "testUsageMetadata", + "ChatMistralAI", + "Usage metadata tokens is not currently supported." + ); + } +} + +const testClass = new ChatMistralAIStandardIntegrationTests(); + +test("ChatMistralAIStandardIntegrationTests", async () => { + const testResults = await testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-mistralai/src/tests/chat_models.standard.test.ts b/libs/langchain-mistralai/src/tests/chat_models.standard.test.ts new file mode 100644 index 000000000000..c925f3dc6d47 --- /dev/null +++ b/libs/langchain-mistralai/src/tests/chat_models.standard.test.ts @@ -0,0 +1,53 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelUnitTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { LangSmithParams } from "@langchain/core/language_models/chat_models"; +import { ChatMistralAI, ChatMistralAICallOptions } from "../chat_models.js"; + +class ChatMistralAIStandardUnitTests extends ChatModelUnitTests< + ChatMistralAICallOptions, + AIMessageChunk +> { + constructor() { + super({ + Cls: ChatMistralAI, + chatModelHasToolCalling: true, + chatModelHasStructuredOutput: true, + constructorArgs: {}, + }); + // This must be set so method like `.bindTools` or `.withStructuredOutput` + // which we call after instantiating the model will work. + // (constructor will throw if API key is not set) + process.env.MISTRAL_API_KEY = "test"; + } + + expectedLsParams(): Partial { + console.warn( + "Overriding testStandardParams. ChatCloudflareWorkersAI does not support stop sequences." + ); + return { + ls_provider: "string", + ls_model_name: "string", + ls_model_type: "chat", + ls_temperature: 0, + ls_max_tokens: 0, + }; + } + + testChatModelInitApiKey() { + // Unset the API key env var here so this test can properly check + // the API key class arg. + process.env.MISTRAL_API_KEY = ""; + super.testChatModelInitApiKey(); + // Re-set the API key env var here so other tests can run properly. + process.env.MISTRAL_API_KEY = "test"; + } +} + +const testClass = new ChatMistralAIStandardUnitTests(); + +test("ChatMistralAIStandardUnitTests", () => { + const testResults = testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-openai/src/tests/azure/chat_models.standard.int.test.ts b/libs/langchain-openai/src/tests/azure/chat_models.standard.int.test.ts new file mode 100644 index 000000000000..cfec5cfe8eb8 --- /dev/null +++ b/libs/langchain-openai/src/tests/azure/chat_models.standard.int.test.ts @@ -0,0 +1,45 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelIntegrationTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { AzureChatOpenAI } from "../../azure/chat_models.js"; +import { ChatOpenAICallOptions } from "../../chat_models.js"; + +class AzureChatOpenAIStandardIntegrationTests extends ChatModelIntegrationTests< + ChatOpenAICallOptions, + AIMessageChunk +> { + constructor() { + super({ + Cls: AzureChatOpenAI, + chatModelHasToolCalling: true, + chatModelHasStructuredOutput: true, + constructorArgs: { + model: "gpt-3.5-turbo", + }, + }); + } + + async testToolMessageHistoriesListContent() { + this.skipTestMessage( + "testToolMessageHistoriesListContent", + "AzureChatOpenAI", + "Not properly implemented." + ); + } + + async testUsageMetadataStreaming() { + this.skipTestMessage( + "testUsageMetadataStreaming", + "AzureChatOpenAI", + "Streaming tokens is not currently supported." + ); + } +} + +const testClass = new AzureChatOpenAIStandardIntegrationTests(); + +test("AzureChatOpenAIStandardIntegrationTests", async () => { + const testResults = await testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-openai/src/tests/azure/chat_models.standard.test.ts b/libs/langchain-openai/src/tests/azure/chat_models.standard.test.ts new file mode 100644 index 000000000000..1dd91e128e1b --- /dev/null +++ b/libs/langchain-openai/src/tests/azure/chat_models.standard.test.ts @@ -0,0 +1,37 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelUnitTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { AzureChatOpenAI } from "../../azure/chat_models.js"; +import { ChatOpenAICallOptions } from "../../chat_models.js"; + +class AzureChatOpenAIStandardUnitTests extends ChatModelUnitTests< + ChatOpenAICallOptions, + AIMessageChunk +> { + constructor() { + super({ + Cls: AzureChatOpenAI, + chatModelHasToolCalling: true, + chatModelHasStructuredOutput: true, + constructorArgs: {}, + }); + process.env.AZURE_OPENAI_API_KEY = "test"; + process.env.AZURE_OPENAI_API_DEPLOYMENT_NAME = "test"; + process.env.AZURE_OPENAI_API_VERSION = "test"; + process.env.AZURE_OPENAI_BASE_PATH = "test"; + } + + testChatModelInitApiKey() { + console.warn( + "AzureChatOpenAI does not require a single API key. Skipping..." + ); + } +} + +const testClass = new AzureChatOpenAIStandardUnitTests(); + +test("AzureChatOpenAIStandardUnitTests", () => { + const testResults = testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-standard-tests/src/base.ts b/libs/langchain-standard-tests/src/base.ts index 504391c2e3ae..fa4d56404795 100644 --- a/libs/langchain-standard-tests/src/base.ts +++ b/libs/langchain-standard-tests/src/base.ts @@ -8,38 +8,78 @@ export type RecordStringAny = Record; export type BaseChatModelConstructor< CallOptions extends BaseChatModelCallOptions = BaseChatModelCallOptions, - OutputMessageType extends BaseMessageChunk = BaseMessageChunk -> = new (...args: any[]) => BaseChatModel; + OutputMessageType extends BaseMessageChunk = BaseMessageChunk, + ConstructorArgs extends RecordStringAny = RecordStringAny +> = new (args: ConstructorArgs) => BaseChatModel< + CallOptions, + OutputMessageType +>; export type BaseChatModelsTestsFields< CallOptions extends BaseChatModelCallOptions = BaseChatModelCallOptions, - OutputMessageType extends BaseMessageChunk = BaseMessageChunk + OutputMessageType extends BaseMessageChunk = BaseMessageChunk, + ConstructorArgs extends RecordStringAny = RecordStringAny > = { - Cls: BaseChatModelConstructor; + Cls: BaseChatModelConstructor< + CallOptions, + OutputMessageType, + ConstructorArgs + >; chatModelHasToolCalling: boolean; chatModelHasStructuredOutput: boolean; - constructorArgs: RecordStringAny; + constructorArgs: ConstructorArgs; }; export class BaseChatModelsTests< CallOptions extends BaseChatModelCallOptions = BaseChatModelCallOptions, - OutputMessageType extends BaseMessageChunk = BaseMessageChunk -> implements BaseChatModelsTestsFields + OutputMessageType extends BaseMessageChunk = BaseMessageChunk, + ConstructorArgs extends RecordStringAny = RecordStringAny +> implements + BaseChatModelsTestsFields { - Cls: BaseChatModelConstructor; + Cls: BaseChatModelConstructor< + CallOptions, + OutputMessageType, + ConstructorArgs + >; chatModelHasToolCalling: boolean; chatModelHasStructuredOutput: boolean; - constructorArgs: RecordStringAny; + constructorArgs: ConstructorArgs; constructor( - fields: BaseChatModelsTestsFields + fields: BaseChatModelsTestsFields< + CallOptions, + OutputMessageType, + ConstructorArgs + > ) { this.Cls = fields.Cls; this.chatModelHasToolCalling = fields.chatModelHasToolCalling; this.chatModelHasStructuredOutput = fields.chatModelHasStructuredOutput; this.constructorArgs = fields.constructorArgs; } + + get multipleApiKeysRequiredMessage(): string { + return "Multiple API keys are required."; + } + + /** + * Log a warning message when skipping a test. + */ + skipTestMessage( + testName: string, + chatClassName: string, + extra?: string + ): void { + console.warn( + { + chatClassName, + reason: extra ?? "n/a", + }, + `Skipping ${testName}.` + ); + } } diff --git a/libs/langchain-standard-tests/src/integration_tests/chat_models.ts b/libs/langchain-standard-tests/src/integration_tests/chat_models.ts index 11c1f9183805..bb249bb30364 100644 --- a/libs/langchain-standard-tests/src/integration_tests/chat_models.ts +++ b/libs/langchain-standard-tests/src/integration_tests/chat_models.ts @@ -10,7 +10,11 @@ import { } from "@langchain/core/messages"; import { z } from "zod"; import { StructuredTool } from "@langchain/core/tools"; -import { BaseChatModelsTests, BaseChatModelsTestsFields } from "../base.js"; +import { + BaseChatModelsTests, + BaseChatModelsTestsFields, + RecordStringAny, +} from "../base.js"; const adderSchema = /* #__PURE__ */ z .object({ @@ -34,12 +38,27 @@ class AdderTool extends StructuredTool { export abstract class ChatModelIntegrationTests< CallOptions extends BaseChatModelCallOptions = BaseChatModelCallOptions, - OutputMessageType extends BaseMessageChunk = BaseMessageChunk -> extends BaseChatModelsTests { + OutputMessageType extends BaseMessageChunk = BaseMessageChunk, + ConstructorArgs extends RecordStringAny = RecordStringAny +> extends BaseChatModelsTests { + functionId = "abc123"; + constructor( - fields: BaseChatModelsTestsFields + fields: BaseChatModelsTestsFields< + CallOptions, + OutputMessageType, + ConstructorArgs + > & { + /** + * The ID to set for function calls. + * Set this field to override the default function ID. + * @default "abc123" + */ + functionId?: string; + } ) { super(fields); + this.functionId = fields.functionId ?? this.functionId; } async testInvoke( @@ -169,7 +188,7 @@ export abstract class ChatModelIntegrationTests< const functionName = adderTool.name; const functionArgs = { a: 1, b: 2 }; - const functionId = "abc123"; + const { functionId } = this; const functionResult = await adderTool.invoke(functionArgs); const messagesStringContent = [ @@ -219,7 +238,7 @@ export abstract class ChatModelIntegrationTests< const functionName = adderTool.name; const functionArgs = { a: 1, b: 2 }; - const functionId = "abc123"; + const { functionId } = this; const functionResult = await adderTool.invoke(functionArgs); const messagesListContent = [ @@ -274,7 +293,7 @@ export abstract class ChatModelIntegrationTests< const functionName = adderTool.name; const functionArgs = { a: 1, b: 2 }; - const functionId = "abc123"; + const { functionId } = this; const functionResult = await adderTool.invoke(functionArgs); const messagesStringContent = [ diff --git a/libs/langchain-standard-tests/src/unit_tests/chat_models.ts b/libs/langchain-standard-tests/src/unit_tests/chat_models.ts index 74fafe0813e9..47de9478fcc2 100644 --- a/libs/langchain-standard-tests/src/unit_tests/chat_models.ts +++ b/libs/langchain-standard-tests/src/unit_tests/chat_models.ts @@ -33,10 +33,15 @@ class PersonTool extends StructuredTool { export abstract class ChatModelUnitTests< CallOptions extends BaseChatModelCallOptions = BaseChatModelCallOptions, - OutputMessageType extends BaseMessageChunk = BaseMessageChunk -> extends BaseChatModelsTests { + OutputMessageType extends BaseMessageChunk = BaseMessageChunk, + ConstructorArgs extends RecordStringAny = RecordStringAny +> extends BaseChatModelsTests { constructor( - fields: BaseChatModelsTestsFields + fields: BaseChatModelsTestsFields< + CallOptions, + OutputMessageType, + ConstructorArgs + > ) { const standardChatModelParams: RecordStringAny = { temperature: 0, @@ -54,6 +59,22 @@ export abstract class ChatModelUnitTests< }); } + /** + * Override this method if the chat model being tested does not + * support all expected LangSmith parameters. + * @returns {Partial} The LangSmith parameters expected by the chat model. + */ + expectedLsParams(): Partial { + return { + ls_provider: "string", + ls_model_name: "string", + ls_model_type: "chat", + ls_temperature: 0, + ls_max_tokens: 0, + ls_stop: ["Array"], + }; + } + testChatModelInit() { const chatModel = new this.Cls(this.constructorArgs); expect(chatModel).toBeDefined(); @@ -88,14 +109,7 @@ export abstract class ChatModelUnitTests< } testStandardParams() { - const expectedParams: LangSmithParams = { - ls_provider: "string", - ls_model_name: "string", - ls_model_type: "chat", - ls_temperature: 0, - ls_max_tokens: 0, - ls_stop: ["Array"], - }; + const expectedParams = this.expectedLsParams(); const chatModel = new this.Cls(this.constructorArgs); const lsParams = chatModel.getLsParams({} as any); diff --git a/yarn.lock b/yarn.lock index 402f675f2586..4dfdd6afcf71 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8964,6 +8964,7 @@ __metadata: "@jest/globals": ^29.5.0 "@langchain/core": ">0.1.0 <0.3.0" "@langchain/scripts": ~0.0.14 + "@langchain/standard-tests": "workspace:*" "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 dpdm: ^3.12.0 @@ -8995,6 +8996,7 @@ __metadata: "@jest/globals": ^29.5.0 "@langchain/core": ">0.1.0 <0.3.0" "@langchain/scripts": ~0.0.14 + "@langchain/standard-tests": "workspace:*" "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 "@tsconfig/recommended": ^1.0.3 @@ -9037,6 +9039,7 @@ __metadata: "@jest/globals": ^29.5.0 "@langchain/core": ">0.1.58 <0.3.0" "@langchain/scripts": ~0.0.14 + "@langchain/standard-tests": "workspace:*" "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 "@tsconfig/recommended": ^1.0.3 @@ -9716,6 +9719,7 @@ __metadata: "@langchain/core": ">0.1.56 <0.3.0" "@langchain/google-common": ~0.0.17 "@langchain/scripts": ~0.0.14 + "@langchain/standard-tests": "workspace:*" "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 "@tsconfig/recommended": ^1.0.3 @@ -9749,6 +9753,7 @@ __metadata: "@jest/globals": ^29.5.0 "@langchain/core": ">0.1.5 <0.3.0" "@langchain/scripts": ~0.0.14 + "@langchain/standard-tests": "workspace:*" "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 "@tsconfig/recommended": ^1.0.3 @@ -9878,6 +9883,7 @@ __metadata: "@langchain/core": ">0.1.56 <0.3.0" "@langchain/openai": "workspace:^" "@langchain/scripts": ~0.0.14 + "@langchain/standard-tests": "workspace:*" "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 "@tsconfig/recommended": ^1.0.3 @@ -9928,6 +9934,7 @@ __metadata: "@jest/globals": ^29.5.0 "@langchain/core": ">0.1.56 <0.3.0" "@langchain/scripts": ~0.0.14 + "@langchain/standard-tests": "workspace:*" "@mistralai/mistralai": ^0.4.0 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29