From 9228b6cb92a89fe25822c99ad45dcb538e2f5862 Mon Sep 17 00:00:00 2001 From: jacoblee93 Date: Tue, 24 Sep 2024 13:42:16 -0700 Subject: [PATCH 1/4] Add support for formatting LangChain prompts as OpenAI and Anthropic payloads --- js/.gitignore | 8 +++ js/package.json | 32 ++++++++++- js/scripts/create-entrypoints.js | 2 + js/src/index.ts | 2 +- js/src/utils/prompts/anthropic.ts | 57 +++++++++++++++++++ js/src/utils/prompts/openai.ts | 52 +++++++++++++++++ .../utils/prompts/tests/anthropic.int.test.ts | 25 ++++++++ js/src/utils/prompts/tests/openai.int.test.ts | 22 +++++++ js/tsconfig.json | 4 +- js/yarn.lock | 52 ++++++++++++++++- 10 files changed, 251 insertions(+), 5 deletions(-) create mode 100644 js/src/utils/prompts/anthropic.ts create mode 100644 js/src/utils/prompts/openai.ts create mode 100644 js/src/utils/prompts/tests/anthropic.int.test.ts create mode 100644 js/src/utils/prompts/tests/openai.int.test.ts diff --git a/js/.gitignore b/js/.gitignore index e758389d2..45799b05b 100644 --- a/js/.gitignore +++ b/js/.gitignore @@ -79,6 +79,14 @@ Chinook_Sqlite.sql /singletons/traceable.js /singletons/traceable.d.ts /singletons/traceable.d.cts +/utils/prompts/anthropic.cjs +/utils/prompts/anthropic.js +/utils/prompts/anthropic.d.ts +/utils/prompts/anthropic.d.cts +/utils/prompts/openai.cjs +/utils/prompts/openai.js +/utils/prompts/openai.d.ts +/utils/prompts/openai.d.cts /index.cjs /index.js /index.d.ts diff --git a/js/package.json b/js/package.json index d54af9b47..c16923708 100644 --- a/js/package.json +++ b/js/package.json @@ -1,6 +1,6 @@ { "name": "langsmith", - "version": "0.1.60", + "version": "0.1.61", "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.", "packageManager": "yarn@1.22.19", "files": [ @@ -53,6 +53,14 @@ "singletons/traceable.js", "singletons/traceable.d.ts", "singletons/traceable.d.cts", + "utils/prompts/anthropic.cjs", + "utils/prompts/anthropic.js", + "utils/prompts/anthropic.d.ts", + "utils/prompts/anthropic.d.cts", + "utils/prompts/openai.cjs", + "utils/prompts/openai.js", + "utils/prompts/openai.d.ts", + "utils/prompts/openai.d.cts", "index.cjs", "index.js", "index.d.ts", @@ -106,12 +114,14 @@ }, "devDependencies": { "@ai-sdk/openai": "^0.0.40", + "@anthropic-ai/sdk": "^0.27.3", "@babel/preset-env": "^7.22.4", "@faker-js/faker": "^8.4.1", "@jest/globals": "^29.5.0", + "@langchain/anthropic": "^0.3.2", "@langchain/core": "^0.3.1", "@langchain/langgraph": "^0.2.3", - "@langchain/openai": "^0.3.0", + "@langchain/openai": "^0.3.1", "@tsconfig/recommended": "^1.0.2", "@types/jest": "^29.5.1", "@typescript-eslint/eslint-plugin": "^5.59.8", @@ -266,6 +276,24 @@ "import": "./singletons/traceable.js", "require": "./singletons/traceable.cjs" }, + "./utils/prompts/anthropic": { + "types": { + "import": "./utils/prompts/anthropic.d.ts", + "require": "./utils/prompts/anthropic.d.cts", + "default": "./utils/prompts/anthropic.d.ts" + }, + "import": "./utils/prompts/anthropic.js", + "require": "./utils/prompts/anthropic.cjs" + }, + "./utils/prompts/openai": { + "types": { + "import": "./utils/prompts/openai.d.ts", + "require": "./utils/prompts/openai.d.cts", + "default": "./utils/prompts/openai.d.ts" + }, + "import": "./utils/prompts/openai.js", + "require": "./utils/prompts/openai.cjs" + }, "./package.json": "./package.json" } } diff --git a/js/scripts/create-entrypoints.js b/js/scripts/create-entrypoints.js index a3487f756..a3393cc1e 100644 --- a/js/scripts/create-entrypoints.js +++ b/js/scripts/create-entrypoints.js @@ -19,6 +19,8 @@ const entrypoints = { "wrappers/openai": "wrappers/openai", "wrappers/vercel": "wrappers/vercel", "singletons/traceable": "singletons/traceable", + "utils/prompts/anthropic": "utils/prompts/anthropic", + "utils/prompts/openai": "utils/prompts/openai" }; const updateJsonFile = (relativePath, updateFunction) => { diff --git a/js/src/index.ts b/js/src/index.ts index d77c8b7d1..96d782360 100644 --- a/js/src/index.ts +++ b/js/src/index.ts @@ -14,4 +14,4 @@ export { RunTree, type RunTreeConfig } from "./run_trees.js"; export { overrideFetchImplementation } from "./singletons/fetch.js"; // Update using yarn bump-version -export const __version__ = "0.1.60"; +export const __version__ = "0.1.61"; diff --git a/js/src/utils/prompts/anthropic.ts b/js/src/utils/prompts/anthropic.ts new file mode 100644 index 000000000..12eb52dbc --- /dev/null +++ b/js/src/utils/prompts/anthropic.ts @@ -0,0 +1,57 @@ +/* eslint-disable import/no-extraneous-dependencies */ +import type { BasePromptValue } from "@langchain/core/prompt_values"; +import * as langChainAnthropicImports from "@langchain/anthropic"; +import Anthropic from "@anthropic-ai/sdk"; + +/** + * Convert a formatted LangChain prompt (e.g. pulled from the hub) into + * a format expected by Anthropic's JS SDK. + * + * Requires the "@langchain/anthropic" package to be installed in addition + * to the Anthropic SDK. + * + * @example + * ```ts + * import { convertPromptToAnthropic } from "langsmith/utils/hub/anthropic"; + * import { pull } from "langchain/hub"; + * + * import Anthropic from '@anthropic-ai/sdk'; + * + * const prompt = await pull("jacob/joke-generator"); + * const formattedPrompt = await prompt.invoke({ + * topic: "cats", + * }); + * + * const { system, messages } = convertPromptToAnthropic(formattedPrompt); + * + * const anthropicClient = new Anthropic({ + * apiKey: 'your_api_key', + * }); + * + * const anthropicResponse = await anthropicClient.messages.create({ + * model: "claude-3-5-sonnet-20240620", + * max_tokens: 1024, + * stream: false, + * system, + * messages, + * }); + * ``` + * @param formattedPrompt + * @returns A partial Anthropic payload. + */ +export function convertPromptToAnthropic( + formattedPrompt: BasePromptValue +): Anthropic.Messages.MessageCreateParams { + const messages = formattedPrompt.toChatMessages(); + const { _convertMessagesToAnthropicPayload } = langChainAnthropicImports; + if (typeof _convertMessagesToAnthropicPayload !== "function") { + throw new Error( + `Please update your version of "@langchain/anthropic" to 0.3.2 or higher.` + ); + } + const anthropicBody = _convertMessagesToAnthropicPayload(messages); + if (anthropicBody.messages === undefined) { + anthropicBody.messages = []; + } + return anthropicBody; +} diff --git a/js/src/utils/prompts/openai.ts b/js/src/utils/prompts/openai.ts new file mode 100644 index 000000000..38329d94d --- /dev/null +++ b/js/src/utils/prompts/openai.ts @@ -0,0 +1,52 @@ +/* eslint-disable import/no-extraneous-dependencies */ +import type { BasePromptValue } from "@langchain/core/prompt_values"; +import * as langChainOpenAIImports from "@langchain/openai"; +import type { OpenAI } from "openai"; + +/** + * Convert a formatted LangChain prompt (e.g. pulled from the hub) into + * a format expected by OpenAI's JS SDK. + * + * Requires the "@langchain/openai" package to be installed in addition + * to the OpenAI SDK. + * + * @example + * ```ts + * import { convertPromptToOpenAI } from "langsmith/utils/hub/openai"; + * import { pull } from "langchain/hub"; + * + * import OpenAI from 'openai'; + * + * const prompt = await pull("jacob/joke-generator"); + * const formattedPrompt = await prompt.invoke({ + * topic: "cats", + * }); + * + * const { messages } = convertPromptToOpenAI(formattedPrompt); + * + * const openAIClient = new OpenAI(); + * + * const openaiResponse = await openAIClient.chat.completions.create({ + * model: "gpt-4o", + * messages, + * }); + * ``` + * @param formattedPrompt + * @returns A partial OpenAI payload. + */ +export function convertPromptToOpenAI(formattedPrompt: BasePromptValue): { + messages: OpenAI.Chat.ChatCompletionMessageParam[]; +} { + const messages = formattedPrompt.toChatMessages(); + const { _convertMessagesToOpenAIParams } = langChainOpenAIImports; + if (typeof _convertMessagesToOpenAIParams !== "function") { + throw new Error( + `Please update your version of "@langchain/openai" to 0.3.1 or higher.` + ); + } + return { + messages: _convertMessagesToOpenAIParams( + messages + ) as OpenAI.Chat.ChatCompletionMessageParam[], + }; +} diff --git a/js/src/utils/prompts/tests/anthropic.int.test.ts b/js/src/utils/prompts/tests/anthropic.int.test.ts new file mode 100644 index 000000000..9e895498a --- /dev/null +++ b/js/src/utils/prompts/tests/anthropic.int.test.ts @@ -0,0 +1,25 @@ +import Anthropic from "@anthropic-ai/sdk"; +import { pull } from "langchain/hub"; + +import { convertPromptToAnthropic } from "../anthropic.js"; + +test("basic traceable implementation", async () => { + const prompt = await pull("jacob/joke-generator"); + const formattedPrompt = await prompt.invoke({ + topic: "cats", + }); + + const { system, messages } = convertPromptToAnthropic(formattedPrompt); + + const anthropicClient = new Anthropic(); + + const anthropicResponse = await anthropicClient.messages.create({ + model: "claude-3-5-sonnet-20240620", + system, + messages: messages, + max_tokens: 1024, + stream: false, + }); + + console.log(anthropicResponse); +}); diff --git a/js/src/utils/prompts/tests/openai.int.test.ts b/js/src/utils/prompts/tests/openai.int.test.ts new file mode 100644 index 000000000..890e2b33b --- /dev/null +++ b/js/src/utils/prompts/tests/openai.int.test.ts @@ -0,0 +1,22 @@ +import OpenAI from "openai"; +import { pull } from "langchain/hub"; + +import { convertPromptToOpenAI } from "../openai.js"; + +test("basic traceable implementation", async () => { + const prompt = await pull("jacob/joke-generator"); + const formattedPrompt = await prompt.invoke({ + topic: "cats", + }); + + const { messages } = convertPromptToOpenAI(formattedPrompt); + + const openAIClient = new OpenAI(); + + const openAIResponse = await openAIClient.chat.completions.create({ + model: "gpt-4o-mini", + messages, + }); + + console.log(openAIResponse); +}); diff --git a/js/tsconfig.json b/js/tsconfig.json index ab24d6247..1af36fd96 100644 --- a/js/tsconfig.json +++ b/js/tsconfig.json @@ -43,7 +43,9 @@ "src/anonymizer/index.ts", "src/wrappers/openai.ts", "src/wrappers/vercel.ts", - "src/singletons/traceable.ts" + "src/singletons/traceable.ts", + "src/utils/prompts/anthropic.ts", + "src/utils/prompts/openai.ts" ] } } diff --git a/js/yarn.lock b/js/yarn.lock index 2d3032272..39a190399 100644 --- a/js/yarn.lock +++ b/js/yarn.lock @@ -78,6 +78,19 @@ "@jridgewell/gen-mapping" "^0.3.0" "@jridgewell/trace-mapping" "^0.3.9" +"@anthropic-ai/sdk@^0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@anthropic-ai/sdk/-/sdk-0.27.3.tgz#592cdd873c85ffab9589ae6f2e250cbf150e1475" + integrity sha512-IjLt0gd3L4jlOfilxVXTifn42FnVffMgDC04RJK1KDZpmkBWLv0XC92MVVmkxrFZNS/7l3xWgP/I3nqtX1sQHw== + dependencies: + "@types/node" "^18.11.18" + "@types/node-fetch" "^2.6.4" + abort-controller "^3.0.0" + agentkeepalive "^4.2.1" + form-data-encoder "1.7.2" + formdata-node "^4.3.2" + node-fetch "^2.6.7" + "@babel/code-frame@^7.0.0", "@babel/code-frame@^7.12.13", "@babel/code-frame@^7.21.4", "@babel/code-frame@^7.22.13": version "7.22.13" resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.22.13.tgz#e3c1c099402598483b7a8c46a721d1038803755e" @@ -1368,6 +1381,16 @@ "@jridgewell/resolve-uri" "3.1.0" "@jridgewell/sourcemap-codec" "1.4.14" +"@langchain/anthropic@^0.3.2": + version "0.3.2" + resolved "https://registry.yarnpkg.com/@langchain/anthropic/-/anthropic-0.3.2.tgz#ca28576573c5b2b9d2277f959100996603a2b977" + integrity sha512-Bgb0SyxQcX+/GOGQ66RsmNmNdnXwpvQt9HLNnwPOSDmgJIegzst3KpB/iHbckgiWtHXBE2ETzWqkLR38/kfHpQ== + dependencies: + "@anthropic-ai/sdk" "^0.27.3" + fast-xml-parser "^4.4.1" + zod "^3.22.4" + zod-to-json-schema "^3.22.4" + "@langchain/core@^0.3.1": version "0.3.1" resolved "https://registry.yarnpkg.com/@langchain/core/-/core-0.3.1.tgz#f06206809575b2a95eaef609b3273842223c0786" @@ -1402,7 +1425,7 @@ uuid "^10.0.0" zod "^3.23.8" -"@langchain/openai@>=0.1.0 <0.4.0", "@langchain/openai@^0.3.0": +"@langchain/openai@>=0.1.0 <0.4.0": version "0.3.0" resolved "https://registry.yarnpkg.com/@langchain/openai/-/openai-0.3.0.tgz#89329ab9350187269a471dac2c2f4fca5f1fc5a3" integrity sha512-yXrz5Qn3t9nq3NQAH2l4zZOI4ev2CFdLC5kvmi5SdW4bggRuM40SXTUAY3VRld4I5eocYfk82VbrlA+6dvN5EA== @@ -1412,6 +1435,16 @@ zod "^3.22.4" zod-to-json-schema "^3.22.3" +"@langchain/openai@^0.3.1": + version "0.3.1" + resolved "https://registry.yarnpkg.com/@langchain/openai/-/openai-0.3.1.tgz#3841cf992f4da571514798ee5abf8e20cb66933f" + integrity sha512-+BEIs8zw4QJE9RYce4K0oPScWYZjQJ+MKgNqS3/kEc2lRBdw2061yy0raO6/HW6+ir3qUh8oABT/5BNuzSkgkw== + dependencies: + js-tiktoken "^1.0.12" + openai "^4.57.3" + zod "^3.22.4" + zod-to-json-schema "^3.22.3" + "@langchain/textsplitters@>=0.0.0 <0.2.0": version "0.1.0" resolved "https://registry.yarnpkg.com/@langchain/textsplitters/-/textsplitters-0.1.0.tgz#f37620992192df09ecda3dfbd545b36a6bcbae46" @@ -2669,6 +2702,13 @@ fast-levenshtein@^2.0.6: resolved "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz" integrity sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw== +fast-xml-parser@^4.4.1: + version "4.5.0" + resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.5.0.tgz#2882b7d01a6825dfdf909638f2de0256351def37" + integrity sha512-/PlTQCI96+fZMAOLMZK4CWG1ItCbfZ/0jx7UIJFChPNrx7tcEgerUgWbeieCM9MfHInUDyK8DWYZ+YrywDJuTg== + dependencies: + strnum "^1.0.5" + fastq@^1.6.0: version "1.15.0" resolved "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz" @@ -4488,6 +4528,11 @@ strip-json-comments@^3.1.0, strip-json-comments@^3.1.1: resolved "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz" integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig== +strnum@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/strnum/-/strnum-1.0.5.tgz#5c4e829fe15ad4ff0d20c3db5ac97b73c9b072db" + integrity sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA== + supports-color@^5.3.0: version "5.5.0" resolved "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz" @@ -4881,6 +4926,11 @@ zod-to-json-schema@^3.22.3: resolved "https://registry.yarnpkg.com/zod-to-json-schema/-/zod-to-json-schema-3.22.4.tgz#f8cc691f6043e9084375e85fb1f76ebafe253d70" integrity sha512-2Ed5dJ+n/O3cU383xSY28cuVi0BCQhF8nYqWU5paEpl7fVdqdAmiLdqLyfblbNdfOFwFfi/mqU4O1pwc60iBhQ== +zod-to-json-schema@^3.22.4: + version "3.23.3" + resolved "https://registry.yarnpkg.com/zod-to-json-schema/-/zod-to-json-schema-3.23.3.tgz#56cf4e0bd5c4096ab46e63159e20998ec7b19c39" + integrity sha512-TYWChTxKQbRJp5ST22o/Irt9KC5nj7CdBKYB/AosCRdj/wxEMvv4NNaj9XVUHDOIp53ZxArGhnw5HMZziPFjog== + zod@^3.22.4: version "3.22.4" resolved "https://registry.yarnpkg.com/zod/-/zod-3.22.4.tgz#f31c3a9386f61b1f228af56faa9255e845cf3fff" From 87bf3d773581d8679305910242d531419c6890a9 Mon Sep 17 00:00:00 2001 From: jacoblee93 Date: Tue, 24 Sep 2024 13:48:30 -0700 Subject: [PATCH 2/4] Use Haiku --- js/src/utils/prompts/tests/anthropic.int.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js/src/utils/prompts/tests/anthropic.int.test.ts b/js/src/utils/prompts/tests/anthropic.int.test.ts index 9e895498a..a5524677b 100644 --- a/js/src/utils/prompts/tests/anthropic.int.test.ts +++ b/js/src/utils/prompts/tests/anthropic.int.test.ts @@ -14,7 +14,7 @@ test("basic traceable implementation", async () => { const anthropicClient = new Anthropic(); const anthropicResponse = await anthropicClient.messages.create({ - model: "claude-3-5-sonnet-20240620", + model: "claude-3-haiku-20240307", system, messages: messages, max_tokens: 1024, From 0e96b1ea87914d9c58db2c11367ef543ca57a844 Mon Sep 17 00:00:00 2001 From: jacoblee93 Date: Tue, 24 Sep 2024 13:49:57 -0700 Subject: [PATCH 3/4] Pull Anthropic key into CI --- .github/actions/js-integration-tests/action.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/actions/js-integration-tests/action.yml b/.github/actions/js-integration-tests/action.yml index fa55acf09..52a7540bc 100644 --- a/.github/actions/js-integration-tests/action.yml +++ b/.github/actions/js-integration-tests/action.yml @@ -13,6 +13,9 @@ inputs: openai-api-key: description: "OpenAI API key" required: false + anthropic-api-key: + description: "Anthropic API key" + required: false runs: using: "composite" steps: @@ -36,6 +39,7 @@ runs: shell: bash working-directory: js env: + ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }} LANGCHAIN_TRACING_V2: "true" LANGCHAIN_ENDPOINT: ${{ inputs.langchain-endpoint }} LANGCHAIN_API_KEY: ${{ inputs.langchain-api-key }} From ab4accbdf8895a0378892d4ccf203f1f86bf5741 Mon Sep 17 00:00:00 2001 From: jacoblee93 Date: Tue, 24 Sep 2024 13:52:25 -0700 Subject: [PATCH 4/4] Use assertions in tests --- js/src/utils/prompts/tests/anthropic.int.test.ts | 2 +- js/src/utils/prompts/tests/openai.int.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/js/src/utils/prompts/tests/anthropic.int.test.ts b/js/src/utils/prompts/tests/anthropic.int.test.ts index a5524677b..d057a561c 100644 --- a/js/src/utils/prompts/tests/anthropic.int.test.ts +++ b/js/src/utils/prompts/tests/anthropic.int.test.ts @@ -21,5 +21,5 @@ test("basic traceable implementation", async () => { stream: false, }); - console.log(anthropicResponse); + expect(anthropicResponse.content).toBeDefined(); }); diff --git a/js/src/utils/prompts/tests/openai.int.test.ts b/js/src/utils/prompts/tests/openai.int.test.ts index 890e2b33b..48b12f715 100644 --- a/js/src/utils/prompts/tests/openai.int.test.ts +++ b/js/src/utils/prompts/tests/openai.int.test.ts @@ -18,5 +18,5 @@ test("basic traceable implementation", async () => { messages, }); - console.log(openAIResponse); + expect(openAIResponse.choices.length).toBeGreaterThan(0); });