From e49153645dc134c83245b9a0cb15fb9b2c6b601e Mon Sep 17 00:00:00 2001 From: Harrison Chase Date: Sun, 3 Mar 2024 15:30:57 -0800 Subject: [PATCH] add metadata to example, allow for creating feedback on the session level (#486) Co-authored-by: William Fu-Hinthorn <13333726+hinthornw@users.noreply.github.com> --- python/langsmith/client.py | 37 +++++++++++++++++++++++++++++++++---- python/langsmith/schemas.py | 5 ++++- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 7176862d3..a4f96614d 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -2482,6 +2482,7 @@ def create_examples( *, inputs: Sequence[Mapping[str, Any]], outputs: Optional[Sequence[Optional[Mapping[str, Any]]]] = None, + metadata: Optional[Sequence[Optional[Mapping[str, Any]]]] = None, dataset_id: Optional[ID_TYPE] = None, dataset_name: Optional[str] = None, **kwargs: Any, @@ -2494,6 +2495,8 @@ def create_examples( The input values for the examples. outputs : Optional[Sequence[Optional[Mapping[str, Any]]]], default=None The output values for the examples. + metadata : Optional[Sequence[Optional[Mapping[str, Any]]]], default=None + The metadata for the examples. dataset_id : Optional[ID_TYPE], default=None The ID of the dataset to create the examples in. dataset_name : Optional[str], default=None @@ -2518,8 +2521,13 @@ def create_examples( "inputs": in_, "outputs": out_, "dataset_id": dataset_id, + "metadata": metadata_, } - for in_, out_ in zip(inputs, outputs or [None] * len(inputs)) + for in_, out_, metadata_ in zip( + inputs, + outputs or [None] * len(inputs), + metadata or [None] * len(inputs), + ) ] response = self.session.post( @@ -2537,6 +2545,7 @@ def create_example( dataset_name: Optional[str] = None, created_at: Optional[datetime.datetime] = None, outputs: Optional[Mapping[str, Any]] = None, + metadata: Optional[Mapping[str, Any]] = None, example_id: Optional[ID_TYPE] = None, ) -> ls_schemas.Example: """Create a dataset example in the LangSmith API. @@ -2556,6 +2565,8 @@ def create_example( The creation timestamp of the example. outputs : Mapping[str, Any] or None, default=None The output values for the example. + metadata : Mapping[str, Any] or None, default=None + The metadata for the example. exemple_id : UUID or None, default=None The ID of the example to create. If not provided, a new example will be created. @@ -2570,6 +2581,7 @@ def create_example( "inputs": inputs, "outputs": outputs, "dataset_id": dataset_id, + "metadata": metadata, } if created_at: data["created_at"] = created_at.isoformat() @@ -2589,7 +2601,9 @@ def create_example( _tenant_id=self._get_optional_tenant_id(), ) - def read_example(self, example_id: ID_TYPE) -> ls_schemas.Example: + def read_example( + self, example_id: ID_TYPE, *, as_of: Optional[datetime.datetime] = None + ) -> ls_schemas.Example: """Read an example from the LangSmith API. Args: @@ -2600,6 +2614,9 @@ def read_example(self, example_id: ID_TYPE) -> ls_schemas.Example: """ response = self._get_with_retries( f"/examples/{_as_uuid(example_id, 'example_id')}", + params={ + "as_of": as_of.isoformat() if as_of else None, + }, ) return ls_schemas.Example( **response.json(), @@ -2659,6 +2676,7 @@ def update_example( *, inputs: Optional[Dict[str, Any]] = None, outputs: Optional[Mapping[str, Any]] = None, + metadata: Optional[Dict] = None, dataset_id: Optional[ID_TYPE] = None, ) -> Dict[str, Any]: """Update a specific example. @@ -2683,6 +2701,7 @@ def update_example( inputs=inputs, outputs=outputs, dataset_id=dataset_id, + metadata=metadata, ) response = self.session.patch( f"{self.api_url}/examples/{_as_uuid(example_id, 'example_id')}", @@ -2909,7 +2928,7 @@ async def aevaluate_run( def create_feedback( self, - run_id: ID_TYPE, + run_id: Optional[ID_TYPE], key: str, *, score: Union[float, int, bool, None] = None, @@ -2924,6 +2943,7 @@ def create_feedback( feedback_id: Optional[ID_TYPE] = None, feedback_config: Optional[ls_schemas.FeedbackConfig] = None, stop_after_attempt: int = 10, + project_id: Optional[ID_TYPE] = None, **kwargs: Any, ) -> ls_schemas.Feedback: """Create a feedback in the LangSmith API. @@ -2931,7 +2951,8 @@ def create_feedback( Parameters ---------- run_id : str or UUID - The ID of the run to provide feedback on. + The ID of the run to provide feedback for. Either the run_id OR + the project_id must be provided. key : str The name of the metric, tag, or 'aspect' this feedback is about. score : float or int or bool or None, default=None @@ -2958,7 +2979,14 @@ def create_feedback( or freeform. stop_after_attempt : int, default=10 The number of times to retry the request before giving up. + project_id : str or UUID + The ID of the project_id to provide feedback on. One - and only one - of + this and run_id must be provided. """ + if run_id is None and project_id is None: + raise ValueError("One of run_id and project_id must be provided") + if run_id is not None and project_id is not None: + raise ValueError("Only one of run_id and project_id must be provided") if kwargs: warnings.warn( "The following arguments are no longer used in the create_feedback" @@ -3006,6 +3034,7 @@ def create_feedback( created_at=datetime.datetime.now(datetime.timezone.utc), modified_at=datetime.datetime.now(datetime.timezone.utc), feedback_config=feedback_config, + session_id=project_id, ) self.request_with_retries( "POST", diff --git a/python/langsmith/schemas.py b/python/langsmith/schemas.py index c5d5bce7d..695f83fcb 100644 --- a/python/langsmith/schemas.py +++ b/python/langsmith/schemas.py @@ -49,6 +49,7 @@ class ExampleBase(BaseModel): dataset_id: UUID inputs: Dict[str, Any] outputs: Optional[Dict[str, Any]] = Field(default=None) + metadata: Optional[Dict[str, Any]] = Field(default=None) class Config: """Configuration class for the schema.""" @@ -406,7 +407,7 @@ class FeedbackBase(BaseModel): """The time the feedback was created.""" modified_at: Optional[datetime] = None """The time the feedback was last modified.""" - run_id: UUID + run_id: Optional[UUID] """The associated run ID this feedback is logged for.""" key: str """The metric name, tag, or aspect to provide feedback on.""" @@ -420,6 +421,8 @@ class FeedbackBase(BaseModel): """Correction for the run.""" feedback_source: Optional[FeedbackSourceBase] = None """The source of the feedback.""" + session_id: Optional[UUID] = None + """The associated project ID (Session = Project) this feedback is logged for.""" class Config: """Configuration class for the schema."""