Add presigned feedback method (#482)

js/src/client.ts

@@ -9,12 +9,15 @@ import {
   ExampleCreate,
   ExampleUpdate,
   Feedback,
+  FeedbackConfig,
+  FeedbackIngestToken,
   KVMap,
   LangChainBaseMessage,
   Run,
   RunCreate,
   RunUpdate,
   ScoreType,
+  TimeDelta,
   TracerSession,
   TracerSessionResult,
   ValueType,
@@ -159,6 +162,7 @@ interface FeedbackCreate {
   correction?: object | null;
   comment?: string | null;
   feedback_source?: feedback_source | KVMap | null;
+  feedbackConfig?: FeedbackConfig;
 }
 
 interface FeedbackUpdate {
@@ -2039,14 +2043,15 @@ export class Client {
       feedbackSourceType = "api",
       sourceRunId,
       feedbackId,
-      eager = false,
+      feedbackConfig,
     }: {
       score?: ScoreType;
       value?: ValueType;
       correction?: object;
       comment?: string;
       sourceInfo?: object;
       feedbackSourceType?: FeedbackSourceType;
+      feedbackConfig?: FeedbackConfig;
       sourceRunId?: string;
       feedbackId?: string;
       eager?: boolean;
@@ -2078,8 +2083,9 @@ export class Client {
       correction,
       comment,
       feedback_source: feedback_source,
+      feedbackConfig,
     };
-    const url = `${this.apiUrl}/feedback` + (eager ? "/eager" : "");
+    const url = `${this.apiUrl}/feedback`;
     const response = await this.caller.call(fetch, url, {
       method: "POST",
       headers: { ...this.headers, "Content-Type": "application/json" },
@@ -2184,4 +2190,79 @@ export class Client {
       yield* feedbacks;
     }
   }
+
+  /**
+   * Creates a presigned feedback token and URL.
+   *
+   * The token can be used to authorize feedback metrics without
+   * needing an API key. This is useful for giving browser-based
+   * applications the ability to submit feedback without needing
+   * to expose an API key.
+   *
+   * @param runId - The ID of the run.
+   * @param feedbackKey - The feedback key.
+   * @param options - Additional options for the token.
+   * @param options.expiration - The expiration time for the token.
+   *
+   * @returns A promise that resolves to a FeedbackIngestToken.
+   */
+  public async createPresignedFeedbackToken(
+    runId: string,
+    feedbackKey: string,
+    {
+      expiration,
+      feedbackConfig,
+    }: {
+      expiration?: string | TimeDelta;
+      feedbackConfig?: FeedbackConfig;
+    } = {}
+  ): Promise<FeedbackIngestToken> {
+    const body: KVMap = {
+      run_id: runId,
+      feedback_key: feedbackKey,
+      feedback_config: feedbackConfig,
+    };
+    if (expiration) {
+      if (typeof expiration === "string") {
+        body["expires_at"] = expiration;
+      } else if (expiration?.hours || expiration?.minutes || expiration?.days) {
+        body["expires_in"] = expiration;
+      }
+    } else {
+      body["expires_in"] = {
+        hours: 3,
+      };
+    }
+
+    const response = await this.caller.call(
+      fetch,
+      `${this.apiUrl}/feedback/tokens`,
+      {
+        method: "POST",
+        headers: { ...this.headers, "Content-Type": "application/json" },
+        body: JSON.stringify(body),
+        signal: AbortSignal.timeout(this.timeout_ms),
+      }
+    );
+    const result = await response.json();
+    return result as FeedbackIngestToken;
+  }
+
+  /**
+   * Retrieves a list of presigned feedback tokens for a given run ID.
+   * @param runId The ID of the run.
+   * @returns An async iterable of FeedbackIngestToken objects.
+   */
+  public async *listPresignedFeedbackTokens(
+    runId: string
+  ): AsyncIterable<FeedbackIngestToken> {
+    assertUuid(runId);
+    const params = new URLSearchParams({ run_id: runId });
+    for await (const tokens of this._getPaginated<FeedbackIngestToken>(
+      "/feedback/tokens",
+      params
+    )) {
+      yield* tokens;
+    }
+  }
 }

js/src/schemas.ts

@@ -282,3 +282,48 @@ export interface LangChainBaseMessage {
   content: string;
   additional_kwargs?: KVMap;
 }
+
+export interface FeedbackIngestToken {
+  id: string;
+  url: string;
+  expires_at: string;
+}
+
+export interface TimeDelta {
+  days?: number;
+  hours?: number;
+  minutes?: number;
+}
+
+export interface FeedbackCategory {
+  value: number;
+  label?: string | null;
+}
+
+/**
+ * Represents the configuration for feedback.
+ * This determines how the LangSmith service interprets feedback
+ * values of the associated key.
+ */
+export interface FeedbackConfig {
+  /**
+   * The type of feedback.
+   */
+  type: "continuous" | "categorical" | "freeform";
+
+  /**
+   * The minimum value for continuous feedback.
+   */
+  min?: number | null;
+
+  /**
+   * The maximum value for continuous feedback.
+   */
+  max?: number | null;
+
+  /**
+   * If feedback is categorical, this defines the valid categories the server will accept.
+   * Not applicable to continuous or freeform feedback types.
+   */
+  categories?: FeedbackCategory[] | null;
+}

python/langsmith/client.py

@@ -16,6 +16,7 @@
 import threading
 import time
 import uuid
+import warnings
 import weakref
 from dataclasses import dataclass, field
 from queue import Empty, PriorityQueue, Queue
@@ -2921,8 +2922,9 @@ def create_feedback(
         ] = ls_schemas.FeedbackSourceType.API,
         source_run_id: Optional[ID_TYPE] = None,
         feedback_id: Optional[ID_TYPE] = None,
-        eager: bool = False,
+        feedback_config: Optional[ls_schemas.FeedbackConfig] = None,
         stop_after_attempt: int = 10,
+        **kwargs: Any,
     ) -> ls_schemas.Feedback:
         """Create a feedback in the LangSmith API.
 
@@ -2950,14 +2952,19 @@ def create_feedback(
         feedback_id : str or UUID or None, default=None
             The ID of the feedback to create. If not provided, a random UUID will be
             generated.
-        eager : bool, default=False
-            Whether to skip the write queue when creating the feedback. This means
-            that the feedback will be immediately available for reading, but may
-            cause the write to fail if the API is under heavy load, since the target
-            run_id may have not been created yet.
+        feedback_config: FeedbackConfig or None, default=None,
+            The configuration specifying how to interpret feedback with this key.
+            Examples include continuous (with min/max bounds), categorical,
+            or freeform.
         stop_after_attempt : int, default=10
             The number of times to retry the request before giving up.
         """
+        if kwargs:
+            warnings.warn(
+                "The following arguments are no longer used in the create_feedback"
+                f" endpoint: {sorted(kwargs)}",
+                DeprecationWarning,
+            )
         if not isinstance(feedback_source_type, ls_schemas.FeedbackSourceType):
             feedback_source_type = ls_schemas.FeedbackSourceType(feedback_source_type)
         if feedback_source_type == ls_schemas.FeedbackSourceType.API:
@@ -2998,10 +3005,11 @@ def create_feedback(
             feedback_source=feedback_source,
             created_at=datetime.datetime.now(datetime.timezone.utc),
             modified_at=datetime.datetime.now(datetime.timezone.utc),
+            feedback_config=feedback_config,
         )
         self.request_with_retries(
             "POST",
-            self.api_url + "/feedback" + ("/eager" if eager else ""),
+            self.api_url + "/feedback",
             request_kwargs={
                 "data": _dumps_json(feedback.dict(exclude_none=True)),
                 "headers": {
@@ -3129,6 +3137,89 @@ def delete_feedback(self, feedback_id: ID_TYPE) -> None:
         )
         ls_utils.raise_for_status_with_text(response)
 
+    def create_presigned_feedback_token(
+        self,
+        run_id: ID_TYPE,
+        feedback_key: str,
+        *,
+        expiration: Optional[datetime.datetime | datetime.timedelta] = None,
+        feedback_config: Optional[ls_schemas.FeedbackConfig] = None,
+    ) -> ls_schemas.FeedbackIngestToken:
+        """Create a pre-signed URL to send feedback data to.
+
+        This is useful for giving browser-based clients a way to upload
+        feedback data directly to LangSmith without accessing the
+        API key.
+
+        Args:
+            run_id:
+            feedback_key:
+            expiration: The expiration time of the pre-signed URL.
+                Either a datetime or a timedelta offset from now.
+                Default to 3 hours.
+            feedback_config: FeedbackConfig or None.
+                If creating a feedback_key for the first time,
+                this defines how the metric should be interpreted,
+                such as a continuous score (w/ optional bounds),
+                or distribution over categorical values.
+
+        Returns:
+            The pre-signed URL for uploading feedback data.
+        """
+        body: Dict[str, Any] = {
+            "run_id": run_id,
+            "feedback_key": feedback_key,
+            "feedback_config": feedback_config,
+        }
+        if expiration is None:
+            body["expires_in"] = ls_schemas.TimeDeltaInput(
+                days=0,
+                hours=3,
+                minutes=0,
+            )
+        elif isinstance(expiration, datetime.datetime):
+            body["expires_at"] = expiration.isoformat()
+        elif isinstance(expiration, datetime.timedelta):
+            body["expires_in"] = ls_schemas.TimeDeltaInput(
+                days=expiration.days,
+                hours=expiration.seconds // 3600,
+                minutes=(expiration.seconds // 60) % 60,
+            )
+        else:
+            raise ValueError(f"Unknown expiration type: {type(expiration)}")
+
+        response = self.request_with_retries(
+            "post",
+            f"{self.api_url}/feedback/tokens",
+            {
+                "data": _dumps_json(body),
+                "headers": self._headers,
+            },
+        )
+        ls_utils.raise_for_status_with_text(response)
+        return ls_schemas.FeedbackIngestToken(**response.json())
+
+    def list_presigned_feedback_tokens(
+        self,
+        run_id: ID_TYPE,
+    ) -> Iterator[ls_schemas.FeedbackIngestToken]:
+        """List the feedback ingest tokens for a run.
+
+        Args:
+            run_id: The ID of the run to filter by.
+
+        Yields:
+            FeedbackIngestToken
+                The feedback ingest tokens.
+        """
+        params = {
+            "run_id": _as_uuid(run_id, "run_id"),
+        }
+        yield from (
+            ls_schemas.FeedbackIngestToken(**token)
+            for token in self._get_paginated_list("/feedback/tokens", params=params)
+        )
+
     # Annotation Queue API
 
     def list_annotation_queues(