Update serialization (#563)

Better handle cases where a pydantic object raises an error when trying
to serialize itself

python/Makefile

@@ -16,7 +16,7 @@ doctest:
 	poetry run pytest -n auto --durations=10 --doctest-modules langsmith
 
 lint:
-	poetry run ruff .
+	poetry run ruff check .
 	poetry run mypy .
 	poetry run black . --check
 

python/langsmith/client.py

@@ -180,27 +180,31 @@ def _serialize_json(obj: Any, depth: int = 0) -> Any:
             ("model_dump_json", True),  # Pydantic V2
             ("json", True),  # Pydantic V1
             ("to_json", False),  # dataclass_json
+            ("model_dump", True),  # Pydantic V2 with non-serializable fields
+            ("dict", False),  # Pydantic V1 with non-serializable fields
         ]
-
         for attr, exclude_none in serialization_methods:
             if hasattr(obj, attr) and callable(getattr(obj, attr)):
                 try:
                     method = getattr(obj, attr)
                     json_str = (
                         method(exclude_none=exclude_none) if exclude_none else method()
                     )
-                    return json.loads(json_str)
+                    if isinstance(json_str, str):
+                        return json.loads(json_str)
+                    return orjson.loads(_dumps_json(json_str, depth=depth + 1))
                 except Exception as e:
                     logger.debug(f"Failed to serialize {type(obj)} to JSON: {e}")
-                    return repr(obj)
+                    pass
+        all_attrs = {}
         if hasattr(obj, "__slots__"):
-            all_attrs = {slot: getattr(obj, slot, None) for slot in obj.__slots__}
-        elif hasattr(obj, "__dict__"):
-            all_attrs = vars(obj)
-        else:
-            return repr(obj)
-        filtered = {k: v if v is not obj else repr(v) for k, v in all_attrs.items()}
-        return orjson.loads(_dumps_json(filtered, depth=depth + 1))
+            all_attrs.update({slot: getattr(obj, slot, None) for slot in obj.__slots__})
+        if hasattr(obj, "__dict__"):
+            all_attrs.update(vars(obj))
+        if all_attrs:
+            filtered = {k: v if v is not obj else repr(v) for k, v in all_attrs.items()}
+            return orjson.loads(_dumps_json(filtered, depth=depth + 1))
+        return repr(obj)
     except BaseException as e:
         logger.debug(f"Failed to serialize {type(obj)} to JSON: {e}")
         return repr(obj)

python/langsmith/evaluation/evaluator.py

@@ -3,7 +3,7 @@
 import asyncio
 import uuid
 from abc import abstractmethod
-from typing import Callable, Dict, List, Optional, TypedDict, Union, cast
+from typing import Any, Callable, Dict, List, Optional, TypedDict, Union, cast
 
 try:
     from pydantic.v1 import BaseModel, Field, ValidationError  # type: ignore[import]
@@ -161,9 +161,9 @@ def evaluate_run(
             Union[EvaluationResult, EvaluationResults]: The result of the evaluation.
         """  # noqa: E501
         source_run_id = uuid.uuid4()
-        metadata = {"target_run_id": run.id}
-        if getattr(run, "session_id"):
-            metadata["experiment"] = run.session_id
+        metadata: Dict[str, Any] = {"target_run_id": run.id}
+        if getattr(run, "session_id", None):
+            metadata["experiment"] = str(run.session_id)
         result = self.func(
             run,
             example,
@@ -197,7 +197,7 @@ def __call__(
         return self.evaluate_run(run, example)
 
     def __repr__(self) -> str:
-        """String representation of the DynamicRunEvaluator object."""
+        """Represent the DynamicRunEvaluator object."""
         return f"<DynamicRunEvaluator {getattr(self.func, '__name__')}>"
 
 

python/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "langsmith"
-version = "0.1.34"
+version = "0.1.36"
 description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform."
 authors = ["LangChain <[email protected]>"]
 license = "MIT"
@@ -34,8 +34,8 @@ orjson = "^3.9.14"
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.3.1"
 black = ">=23.3,<25.0"
-mypy = "^1.3.0"
-ruff = "^0.1.5"
+mypy = "^1.9.0"
+ruff = "^0.3.4"
 pydantic = ">=1,<2"
 types-requests = "^2.31.0.1"
 pandas-stubs = "^2.0.1.230501"

python/tests/integration_tests/wrappers/test_openai.py

@@ -86,19 +86,19 @@ def test_completions_sync_api(mock_session: mock.MagicMock, stream: bool):
 
     original_client = openai.Client()
     patched_client = wrap_openai(openai.Client())
-    prompt = ("Say 'Hi i'm ChatGPT' then stop.",)
+    prompt = ("Say 'Foo' then stop.",)
     original = original_client.completions.create(
         model="gpt-3.5-turbo-instruct",
         prompt=prompt,
-        max_tokens=5,
+        max_tokens=3,
         temperature=0,
         seed=42,
         stream=stream,
     )
     patched = patched_client.completions.create(
         model="gpt-3.5-turbo-instruct",
         prompt=prompt,
-        max_tokens=5,
+        max_tokens=3,
         temperature=0,
         seed=42,
         stream=stream,

python/tests/unit_tests/test_client.py

@@ -647,6 +647,13 @@ class MyEnum(str, Enum):
         FOO = "foo"
         BAR = "bar"
 
+    class ClassWithFakeJson:
+        def json(self):
+            raise ValueError("This should not be called")
+
+        def to_json(self) -> dict:
+            return {"foo": "bar"}
+
     @dataclasses_json.dataclass_json
     @dataclasses.dataclass
     class Person:
@@ -710,6 +717,7 @@ class MyNamedTuple(NamedTuple):
         "named_tuple": MyNamedTuple(foo="foo", bar=1),
         "cyclic": CyclicClass(),
         "cyclic2": cycle_2,
+        "fake_json": ClassWithFakeJson(),
     }
     res = orjson.loads(_dumps_json(to_serialize))
     expected = {
@@ -746,6 +754,7 @@ class MyNamedTuple(NamedTuple):
         "cyclic": {"cyclic": "SoCyclic"},
         # We don't really care about this case just want to not err
         "cyclic2": lambda _: True,
+        "fake_json": {"foo": "bar"},
     }
     assert set(expected) == set(res)
     for k, v in expected.items():