From 1bc0fb8be7d057c93f08c5982e1cc693850f4860 Mon Sep 17 00:00:00 2001 From: SN <6432132+samnoyes@users.noreply.github.com> Date: Fri, 19 Jan 2024 10:05:29 -0800 Subject: [PATCH 1/4] Add support to js client for LANGCHAIN_ env variable tracking in metadata and revision_id in createRun --- js/src/client.ts | 16 +++++++++++- js/src/utils/env.ts | 59 ++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 70 insertions(+), 5 deletions(-) diff --git a/js/src/client.ts b/js/src/client.ts index 609bc6c53..c39d3d5d1 100644 --- a/js/src/client.ts +++ b/js/src/client.ts @@ -23,7 +23,11 @@ import { convertLangChainMessageToExample, isLangChainMessage, } from "./utils/messages.js"; -import { getEnvironmentVariable, getRuntimeEnvironment } from "./utils/env.js"; +import { + getEnvironmentVariable, + getLangChainEnvVarsMetadata, + getRuntimeEnvironment, +} from "./utils/env.js"; import { RunEvaluator } from "./evaluation/evaluator.js"; import { __version__ } from "./index.js"; @@ -99,6 +103,7 @@ interface CreateRunParams { child_runs?: RunCreate[]; parent_run_id?: string; project_name?: string; + revision_id?: string; } interface projectOptions { @@ -349,7 +354,9 @@ export class Client { public async createRun(run: CreateRunParams): Promise { const headers = { ...this.headers, "Content-Type": "application/json" }; const extra = run.extra ?? {}; + const metadata = extra.metadata; const runtimeEnv = await getRuntimeEnvironment(); + const envVars = getLangChainEnvVarsMetadata(); const session_name = run.project_name; delete run.project_name; const runCreate: RunCreate = { @@ -361,6 +368,13 @@ export class Client { ...runtimeEnv, ...extra.runtime, }, + metadata: { + ...envVars, + ...(envVars.revision_id || run.revision_id + ? { revision_id: run.revision_id ?? envVars.revision_id } + : {}), + ...metadata, + }, }, }; runCreate.inputs = hideInputs(runCreate.inputs); diff --git a/js/src/utils/env.ts b/js/src/utils/env.ts index d5a15f203..d840a50cd 100644 --- a/js/src/utils/env.ts +++ b/js/src/utils/env.ts @@ -86,7 +86,7 @@ export async function getRuntimeEnvironment(): Promise { /** * Retrieves the LangChain-specific environment variables from the current runtime environment. - * Sensitive keys (containing the word "key") have their values redacted for security. + * Sensitive keys (containing the word "key", "token", or "secret") have their values redacted for security. * * @returns {Record} * - A record of LangChain-specific environment variables. @@ -102,7 +102,12 @@ export function getLangChainEnvVars(): Record { } for (const key in envVars) { - if (key.toLowerCase().includes("key") && typeof envVars[key] === "string") { + if ( + (key.toLowerCase().includes("key") || + key.toLowerCase().includes("secret") || + key.toLowerCase().includes("token")) && + typeof envVars[key] === "string" + ) { const value = envVars[key]; envVars[key] = value.slice(0, 2) + "*".repeat(value.length - 4) + value.slice(-2); @@ -112,6 +117,53 @@ export function getLangChainEnvVars(): Record { return envVars; } +/** + * Retrieves the LangChain-specific metadata from the current runtime environment. + * Sensitive keys (containing the word "key", "token", or "secret") have their values redacted for security. + * + * @returns {Record} + * - A record of LangChain-specific metadata environment variables. + */ +export function getLangChainEnvVarsMetadata(): Record { + const allEnvVars = getEnvironmentVariables() || {}; + const envVars: Record = {}; + const excluded = [ + "LANGCHAIN_API_KEY", + "LANGCHAIN_ENDPOINT", + "LANGCHAIN_TRACING_V2", + "LANGCHAIN_PROJECT", + "LANGCHAIN_SESSION", + ]; + + for (const [key, value] of Object.entries(allEnvVars)) { + if ( + key.startsWith("LANGCHAIN_") && + typeof value === "string" && + !excluded.includes(key) + ) { + envVars[key] = value; + } + } + + for (const key in envVars) { + if ( + (key.toLowerCase().includes("key") || + key.toLowerCase().includes("secret") || + key.toLowerCase().includes("token")) && + typeof envVars[key] === "string" + ) { + const value = envVars[key]; + envVars[key] = + value.slice(0, 2) + "*".repeat(value.length - 4) + value.slice(-2); + } else if (key === "LANGCHAIN_REVISION_ID") { + envVars["revision_id"] = envVars[key]; + delete envVars[key]; + } + } + + return envVars; +} + /** * Retrieves the environment variables from the current runtime environment. * @@ -128,7 +180,7 @@ export function getEnvironmentVariables(): Record | undefined { // eslint-disable-next-line no-process-env if (typeof process !== "undefined" && process.env) { // eslint-disable-next-line no-process-env - Object.entries(process.env).reduce( + return Object.entries(process.env).reduce( (acc: { [key: string]: string }, [key, value]) => { acc[key] = String(value); return acc; @@ -136,7 +188,6 @@ export function getEnvironmentVariables(): Record | undefined { {} ); } - // For browsers and other environments, we may not have direct access to env variables // Return undefined or any other fallback as required. return undefined; From adb29d30ff68079650f355b8e2b42a9035bbcd10 Mon Sep 17 00:00:00 2001 From: SN <6432132+samnoyes@users.noreply.github.com> Date: Fri, 19 Jan 2024 10:34:08 -0800 Subject: [PATCH 2/4] Add tests --- js/src/tests/client.int.test.ts | 50 +++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/js/src/tests/client.int.test.ts b/js/src/tests/client.int.test.ts index 760aa2e22..10f146d69 100644 --- a/js/src/tests/client.int.test.ts +++ b/js/src/tests/client.int.test.ts @@ -507,6 +507,56 @@ test("Test create run with masked inputs/outputs", async () => { expect(Object.keys(run2.outputs ?? {})).toHaveLength(0); }, 10000); +test("Test create run with revision id", async () => { + const langchainClient = new Client({ + apiUrl: "http://localhost:1984", + }); + // eslint-disable-next-line no-process-env + process.env.LANGCHAIN_REVISION_ID = "test_revision_id"; + // eslint-disable-next-line no-process-env + process.env.LANGCHAIN_API_KEY = "fake_api_key"; + // eslint-disable-next-line no-process-env + process.env.LANGCHAIN_OTHER_KEY = "test_other_key"; + const projectName = "__test_create_run_with_revision_id"; + const projects = langchainClient.listProjects(); + for await (const project of projects) { + if (project.name === projectName) { + await langchainClient.deleteProject({ projectName }); + } + } + const runId = "0cc29488-3b1b-4151-9476-30b5c1b24883"; + await langchainClient.createRun({ + id: runId, + project_name: projectName, + name: "test_run", + run_type: "llm", + inputs: { prompt: "hello world" }, + outputs: { generation: "hi there" }, + start_time: new Date().getTime(), + end_time: new Date().getTime(), + }); + + const runId2 = "82f19ed3-256f-4571-a078-2ccf11d0eba3"; + await langchainClient.createRun({ + id: runId2, + project_name: projectName, + name: "test_run_2", + run_type: "llm", + inputs: { messages: "hello world 2" }, + start_time: new Date().getTime(), + revision_id: "different_revision_id", + }); + + const run1 = await langchainClient.readRun(runId); + expect(run1.extra?.metadata?.revision_id).toEqual("test_revision_id"); + expect(run1.extra?.metadata.LANGCHAIN_OTHER_KEY).toEqual("test_other_key"); + expect(run1.extra?.metadata).not.toHaveProperty("LANGCHAIN_API_KEY"); + const run2 = await langchainClient.readRun(runId2); + expect(run2.extra?.metadata?.revision_id).toEqual("different_revision_id"); + expect(run2.extra?.metadata.LANGCHAIN_OTHER_KEY).toEqual("test_other_key"); + expect(run2.extra?.metadata).not.toHaveProperty("LANGCHAIN_API_KEY"); +}, 10000); + describe("createChatExample", () => { it("should convert LangChainBaseMessage objects to examples", async () => { const langchainClient = new Client({ From bd2668baa2ab22c280e57b637bef7576f03aa082 Mon Sep 17 00:00:00 2001 From: SN <6432132+samnoyes@users.noreply.github.com> Date: Fri, 19 Jan 2024 11:01:36 -0800 Subject: [PATCH 3/4] add unit test --- js/src/tests/client.test.ts | 47 +++++++++++++++++++++++++++++++++++++ js/src/utils/env.ts | 27 +++++++-------------- 2 files changed, 56 insertions(+), 18 deletions(-) diff --git a/js/src/tests/client.test.ts b/js/src/tests/client.test.ts index 40317ee2e..09d12716d 100644 --- a/js/src/tests/client.test.ts +++ b/js/src/tests/client.test.ts @@ -1,5 +1,10 @@ import { jest } from "@jest/globals"; import { Client } from "../client.js"; +import { + getEnvironmentVariables, + getLangChainEnvVars, + getLangChainEnvVarsMetadata, +} from "../utils/env.js"; describe("Client", () => { describe("createLLMExample", () => { @@ -118,4 +123,46 @@ describe("Client", () => { expect(result).toBe("https://smith.langchain.com"); }); }); + + describe("env functions", () => { + it("should return the env variables correctly", async () => { + // eslint-disable-next-line no-process-env + process.env.LANGCHAIN_REVISION_ID = "test_revision_id"; + // eslint-disable-next-line no-process-env + process.env.LANGCHAIN_API_KEY = "fake_api_key"; + // eslint-disable-next-line no-process-env + process.env.LANGCHAIN_OTHER_KEY = "test_other_key"; + // eslint-disable-next-line no-process-env + process.env.LANGCHAIN_OTHER_NON_SENSITIVE_METADATA = "test_some_metadata"; + // eslint-disable-next-line no-process-env + process.env.LANGCHAIN_ENDPOINT = "https://example.com"; + // eslint-disable-next-line no-process-env + process.env.SOME_RANDOM_THING = "random"; + + const envVars = getEnvironmentVariables(); + const langchainEnvVars = getLangChainEnvVars(); + const langchainMetadataEnvVars = getLangChainEnvVarsMetadata(); + + expect(envVars).toMatchObject({ + LANGCHAIN_REVISION_ID: "test_revision_id", + LANGCHAIN_API_KEY: "fake_api_key", + LANGCHAIN_OTHER_KEY: "test_other_key", + LANGCHAIN_ENDPOINT: "https://example.com", + SOME_RANDOM_THING: "random", + LANGCHAIN_OTHER_NON_SENSITIVE_METADATA: "test_some_metadata", + }); + expect(langchainEnvVars).toMatchObject({ + LANGCHAIN_REVISION_ID: "test_revision_id", + LANGCHAIN_API_KEY: "fa********ey", + LANGCHAIN_OTHER_KEY: "te**********ey", + LANGCHAIN_ENDPOINT: "https://example.com", + LANGCHAIN_OTHER_NON_SENSITIVE_METADATA: "test_some_metadata", + }); + expect(langchainEnvVars).not.toHaveProperty("SOME_RANDOM_THING"); + expect(langchainMetadataEnvVars).toEqual({ + revision_id: "test_revision_id", + LANGCHAIN_OTHER_NON_SENSITIVE_METADATA: "test_some_metadata", + }); + }); + }); }); diff --git a/js/src/utils/env.ts b/js/src/utils/env.ts index d840a50cd..dea4d3d25 100644 --- a/js/src/utils/env.ts +++ b/js/src/utils/env.ts @@ -139,25 +139,16 @@ export function getLangChainEnvVarsMetadata(): Record { if ( key.startsWith("LANGCHAIN_") && typeof value === "string" && - !excluded.includes(key) + !excluded.includes(key) && + !key.toLowerCase().includes("key") && + !key.toLowerCase().includes("secret") && + !key.toLowerCase().includes("token") ) { - envVars[key] = value; - } - } - - for (const key in envVars) { - if ( - (key.toLowerCase().includes("key") || - key.toLowerCase().includes("secret") || - key.toLowerCase().includes("token")) && - typeof envVars[key] === "string" - ) { - const value = envVars[key]; - envVars[key] = - value.slice(0, 2) + "*".repeat(value.length - 4) + value.slice(-2); - } else if (key === "LANGCHAIN_REVISION_ID") { - envVars["revision_id"] = envVars[key]; - delete envVars[key]; + if (key === "LANGCHAIN_REVISION_ID") { + envVars["revision_id"] = value; + } else { + envVars[key] = value; + } } } From 9c38a6147d57d32519a23499793ca4d712d2e670 Mon Sep 17 00:00:00 2001 From: SN <6432132+samnoyes@users.noreply.github.com> Date: Fri, 19 Jan 2024 11:03:14 -0800 Subject: [PATCH 4/4] remove comment --- js/src/utils/env.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/js/src/utils/env.ts b/js/src/utils/env.ts index dea4d3d25..4c073a796 100644 --- a/js/src/utils/env.ts +++ b/js/src/utils/env.ts @@ -119,7 +119,6 @@ export function getLangChainEnvVars(): Record { /** * Retrieves the LangChain-specific metadata from the current runtime environment. - * Sensitive keys (containing the word "key", "token", or "secret") have their values redacted for security. * * @returns {Record} * - A record of LangChain-specific metadata environment variables.