Merge branch 'main' of https://github.com/langchain-ai/langsmith-sdk …

…into jacob/error_handling

js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.1.56",
+  "version": "0.1.59",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "[email protected]",
   "files": [

js/src/client.ts

@@ -32,6 +32,8 @@ import {
   TracerSession,
   TracerSessionResult,
   ValueType,
+  AnnotationQueue,
+  RunWithAnnotationQueueInfo,
 } from "./schemas.js";
 import {
   convertLangChainMessageToExample,
@@ -3103,6 +3105,208 @@ export class Client {
     return results;
   }
 
+  /**
+   * API for managing annotation queues
+   */
+
+  /**
+   * List the annotation queues on the LangSmith API.
+   * @param options - The options for listing annotation queues
+   * @param options.queueIds - The IDs of the queues to filter by
+   * @param options.name - The name of the queue to filter by
+   * @param options.nameContains - The substring that the queue name should contain
+   * @param options.limit - The maximum number of queues to return
+   * @returns An iterator of AnnotationQueue objects
+   */
+  public async *listAnnotationQueues(
+    options: {
+      queueIds?: string[];
+      name?: string;
+      nameContains?: string;
+      limit?: number;
+    } = {}
+  ): AsyncIterableIterator<AnnotationQueue> {
+    const { queueIds, name, nameContains, limit } = options;
+    const params = new URLSearchParams();
+    if (queueIds) {
+      queueIds.forEach((id, i) => {
+        assertUuid(id, `queueIds[${i}]`);
+        params.append("ids", id);
+      });
+    }
+    if (name) params.append("name", name);
+    if (nameContains) params.append("name_contains", nameContains);
+    params.append(
+      "limit",
+      (limit !== undefined ? Math.min(limit, 100) : 100).toString()
+    );
+
+    let count = 0;
+    for await (const queues of this._getPaginated<AnnotationQueue>(
+      "/annotation-queues",
+      params
+    )) {
+      yield* queues;
+      count++;
+      if (limit !== undefined && count >= limit) break;
+    }
+  }
+
+  /**
+   * Create an annotation queue on the LangSmith API.
+   * @param options - The options for creating an annotation queue
+   * @param options.name - The name of the annotation queue
+   * @param options.description - The description of the annotation queue
+   * @param options.queueId - The ID of the annotation queue
+   * @returns The created AnnotationQueue object
+   */
+  public async createAnnotationQueue(options: {
+    name: string;
+    description?: string;
+    queueId?: string;
+  }): Promise<AnnotationQueue> {
+    const { name, description, queueId } = options;
+    const body = {
+      name,
+      description,
+      id: queueId || uuid.v4(),
+    };
+
+    const response = await this.caller.call(
+      _getFetchImplementation(),
+      `${this.apiUrl}/annotation-queues`,
+      {
+        method: "POST",
+        headers: { ...this.headers, "Content-Type": "application/json" },
+        body: JSON.stringify(
+          Object.fromEntries(
+            Object.entries(body).filter(([_, v]) => v !== undefined)
+          )
+        ),
+        signal: AbortSignal.timeout(this.timeout_ms),
+        ...this.fetchOptions,
+      }
+    );
+    await raiseForStatus(response, "create annotation queue");
+    const data = await response.json();
+    return data as AnnotationQueue;
+  }
+
+  /**
+   * Read an annotation queue with the specified queue ID.
+   * @param queueId - The ID of the annotation queue to read
+   * @returns The AnnotationQueue object
+   */
+  public async readAnnotationQueue(queueId: string): Promise<AnnotationQueue> {
+    // TODO: Replace when actual endpoint is added
+    const queueIteratorResult = await this.listAnnotationQueues({
+      queueIds: [queueId],
+    }).next();
+    if (queueIteratorResult.done) {
+      throw new Error(`Annotation queue with ID ${queueId} not found`);
+    }
+    return queueIteratorResult.value;
+  }
+
+  /**
+   * Update an annotation queue with the specified queue ID.
+   * @param queueId - The ID of the annotation queue to update
+   * @param options - The options for updating the annotation queue
+   * @param options.name - The new name for the annotation queue
+   * @param options.description - The new description for the annotation queue
+   */
+  public async updateAnnotationQueue(
+    queueId: string,
+    options: {
+      name: string;
+      description?: string;
+    }
+  ): Promise<void> {
+    const { name, description } = options;
+    const response = await this.caller.call(
+      _getFetchImplementation(),
+      `${this.apiUrl}/annotation-queues/${assertUuid(queueId, "queueId")}`,
+      {
+        method: "PATCH",
+        headers: { ...this.headers, "Content-Type": "application/json" },
+        body: JSON.stringify({ name, description }),
+        signal: AbortSignal.timeout(this.timeout_ms),
+        ...this.fetchOptions,
+      }
+    );
+    await raiseForStatus(response, "update annotation queue");
+  }
+
+  /**
+   * Delete an annotation queue with the specified queue ID.
+   * @param queueId - The ID of the annotation queue to delete
+   */
+  public async deleteAnnotationQueue(queueId: string): Promise<void> {
+    const response = await this.caller.call(
+      _getFetchImplementation(),
+      `${this.apiUrl}/annotation-queues/${assertUuid(queueId, "queueId")}`,
+      {
+        method: "DELETE",
+        headers: { ...this.headers, Accept: "application/json" },
+        signal: AbortSignal.timeout(this.timeout_ms),
+        ...this.fetchOptions,
+      }
+    );
+    await raiseForStatus(response, "delete annotation queue");
+  }
+
+  /**
+   * Add runs to an annotation queue with the specified queue ID.
+   * @param queueId - The ID of the annotation queue
+   * @param runIds - The IDs of the runs to be added to the annotation queue
+   */
+  public async addRunsToAnnotationQueue(
+    queueId: string,
+    runIds: string[]
+  ): Promise<void> {
+    const response = await this.caller.call(
+      _getFetchImplementation(),
+      `${this.apiUrl}/annotation-queues/${assertUuid(queueId, "queueId")}/runs`,
+      {
+        method: "POST",
+        headers: { ...this.headers, "Content-Type": "application/json" },
+        body: JSON.stringify(
+          runIds.map((id, i) => assertUuid(id, `runIds[${i}]`).toString())
+        ),
+        signal: AbortSignal.timeout(this.timeout_ms),
+        ...this.fetchOptions,
+      }
+    );
+    await raiseForStatus(response, "add runs to annotation queue");
+  }
+
+  /**
+   * Get a run from an annotation queue at the specified index.
+   * @param queueId - The ID of the annotation queue
+   * @param index - The index of the run to retrieve
+   * @returns A Promise that resolves to a RunWithAnnotationQueueInfo object
+   * @throws {Error} If the run is not found at the given index or for other API-related errors
+   */
+  public async getRunFromAnnotationQueue(
+    queueId: string,
+    index: number
+  ): Promise<RunWithAnnotationQueueInfo> {
+    const baseUrl = `/annotation-queues/${assertUuid(queueId, "queueId")}/run`;
+    const response = await this.caller.call(
+      _getFetchImplementation(),
+      `${this.apiUrl}${baseUrl}/${index}`,
+      {
+        method: "GET",
+        headers: this.headers,
+        signal: AbortSignal.timeout(this.timeout_ms),
+        ...this.fetchOptions,
+      }
+    );
+
+    await raiseForStatus(response, "get run from annotation queue");
+    return await response.json();
+  }
+
   protected async _currentTenantIsOwner(owner: string): Promise<boolean> {
     const settings = await this._getSettings();
     return owner == "-" || settings.tenant_handle === owner;
@@ -3228,7 +3432,7 @@ export class Client {
       ListCommitsResponse
     >(
       `/commits/${promptOwnerAndName}/`,
-      {} as URLSearchParams,
+      new URLSearchParams(),
       (res) => res.commits
     )) {
       yield* commits;

js/src/evaluation/_runner.ts

@@ -566,6 +566,7 @@ export class _ExperimentManager {
               : new Date(example.created_at).toISOString(),
           },
           client: fields.client,
+          tracingEnabled: true,
         };
         const evaluatorResponse = await evaluator.evaluateRun(
           run,

js/src/index.ts

@@ -14,4 +14,4 @@ export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 
 // Update using yarn bump-version
-export const __version__ = "0.1.56";
+export const __version__ = "0.1.59";

js/src/run_trees.ts

@@ -146,6 +146,8 @@ class Baggage {
 }
 
 export class RunTree implements BaseRun {
+  private static sharedClient: Client | null = null;
+
   id: string;
   name: RunTreeConfig["name"];
   run_type: string;
@@ -173,7 +175,7 @@ export class RunTree implements BaseRun {
   constructor(originalConfig: RunTreeConfig) {
     const defaultConfig = RunTree.getDefaultConfig();
     const { metadata, ...config } = originalConfig;
-    const client = config.client ?? new Client();
+    const client = config.client ?? RunTree.getSharedClient();
     const dedupedMetadata = {
       ...metadata,
       ...config?.extra?.metadata,
@@ -226,6 +228,13 @@ export class RunTree implements BaseRun {
     };
   }
 
+  private static getSharedClient(): Client {
+    if (!RunTree.sharedClient) {
+      RunTree.sharedClient = new Client();
+    }
+    return RunTree.sharedClient;
+  }
+
   public createChild(config: RunTreeConfig): RunTree {
     const child_execution_order = this.child_execution_order + 1;
 

js/src/schemas.ts

@@ -409,7 +409,7 @@ export type RetrieverOutput = Array<{
 export interface InvocationParamsSchema {
   ls_provider?: string;
   ls_model_name?: string;
-  ls_model_type: "chat" | "text";
+  ls_model_type: "chat" | "llm";
   ls_temperature?: number;
   ls_max_tokens?: number;
   ls_stop?: string[];
@@ -474,3 +474,31 @@ export interface LangSmithSettings {
   created_at: string;
   tenant_handle?: string;
 }
+
+export interface AnnotationQueue {
+  /** The unique identifier of the annotation queue. */
+  id: string;
+
+  /** The name of the annotation queue. */
+  name: string;
+
+  /** An optional description of the annotation queue. */
+  description?: string;
+
+  /** The timestamp when the annotation queue was created. */
+  created_at: string;
+
+  /** The timestamp when the annotation queue was last updated. */
+  updated_at: string;
+
+  /** The ID of the tenant associated with the annotation queue. */
+  tenant_id: string;
+}
+
+export interface RunWithAnnotationQueueInfo extends BaseRun {
+  /** The last time this run was reviewed. */
+  last_reviewed_time?: string;
+
+  /** The time this run was added to the queue. */
+  added_at?: string;
+}