From 39be3c772b7b32ead519bd586c8dabd8143f18aa Mon Sep 17 00:00:00 2001 From: isaac hershenson Date: Mon, 9 Dec 2024 13:14:29 -0800 Subject: [PATCH 1/3] flags in tests --- python/tests/integration_tests/test_client.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/python/tests/integration_tests/test_client.py b/python/tests/integration_tests/test_client.py index 53e940c06..ae866521c 100644 --- a/python/tests/integration_tests/test_client.py +++ b/python/tests/integration_tests/test_client.py @@ -379,6 +379,9 @@ def test_error_surfaced_invalid_uri(uri: str) -> None: def test_upload_examples_multipart(langchain_client: Client): """Test uploading examples with attachments via multipart endpoint.""" + langchain_client._info = { + "instance_flags": {"dataset_examples_multipart_enabled": True} + } dataset_name = "__test_upload_examples_multipart" + uuid4().hex[:4] if langchain_client.has_dataset(dataset_name=dataset_name): langchain_client.delete_dataset(dataset_name=dataset_name) @@ -1245,11 +1248,11 @@ def test_list_examples_attachments_keys(langchain_client: Client) -> None: langchain_client.delete_dataset(dataset_id=dataset.id) -@pytest.mark.skip( - reason="Need to land https://github.com/langchain-ai/langsmith-sdk/pull/1209 first" -) def test_evaluate_with_attachments(langchain_client: Client) -> None: """Test evaluating examples with attachments.""" + langchain_client._info = { + "instance_flags": {"dataset_examples_multipart_enabled": True} + } dataset_name = "__test_evaluate_attachments" + uuid4().hex[:4] # 1. Create dataset dataset = langchain_client.create_dataset( @@ -1305,6 +1308,9 @@ def evaluator(run: Run, example: Example) -> Dict[str, Any]: def test_evaluate_with_no_attachments(langchain_client: Client) -> None: """Test evaluating examples without attachments using a target with attachments.""" + langchain_client._info = { + "instance_flags": {"dataset_examples_multipart_enabled": True} + } dataset_name = "__test_evaluate_no_attachments" + uuid4().hex[:4] dataset = langchain_client.create_dataset( dataset_name, From 5c2c74dbd2bf7abe661e89d8249dd67bc31f5640 Mon Sep 17 00:00:00 2001 From: isaac hershenson Date: Mon, 9 Dec 2024 14:16:55 -0800 Subject: [PATCH 2/3] attachment_urls -> attachments --- python/langsmith/client.py | 12 ++++++------ python/langsmith/evaluation/_arunner.py | 8 ++++---- python/langsmith/evaluation/_runner.py | 8 ++++---- python/langsmith/schemas.py | 2 +- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index a0c98cdcd..82c25e8da 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -3833,13 +3833,13 @@ def read_example( ) example = response.json() - attachment_urls = {} + attachments = {} if example["attachment_urls"]: for key, value in example["attachment_urls"].items(): response = requests.get(value["presigned_url"], stream=True) response.raise_for_status() reader = io.BytesIO(response.content) - attachment_urls[key.split(".")[1]] = { + attachments[key.split(".")[1]] = { "presigned_url": value["presigned_url"], "reader": reader, } @@ -3847,7 +3847,7 @@ def read_example( return ls_schemas.Example( **example, - attachment_urls=attachment_urls, + attachments=attachments, _host_url=self._host_url, _tenant_id=self._get_optional_tenant_id(), ) @@ -3920,13 +3920,13 @@ def list_examples( for i, example in enumerate( self._get_paginated_list("/examples", params=params) ): - attachment_urls = {} + attachments = {} if example["attachment_urls"]: for key, value in example["attachment_urls"].items(): response = requests.get(value["presigned_url"], stream=True) response.raise_for_status() reader = io.BytesIO(response.content) - attachment_urls[key.split(".")[1]] = { + attachments[key.split(".")[1]] = { "presigned_url": value["presigned_url"], "reader": reader, } @@ -3934,7 +3934,7 @@ def list_examples( yield ls_schemas.Example( **example, - attachment_urls=attachment_urls, + attachments=attachments, _host_url=self._host_url, _tenant_id=self._get_optional_tenant_id(), ) diff --git a/python/langsmith/evaluation/_arunner.py b/python/langsmith/evaluation/_arunner.py index 729166add..ccbad302b 100644 --- a/python/langsmith/evaluation/_arunner.py +++ b/python/langsmith/evaluation/_arunner.py @@ -1023,7 +1023,7 @@ def _get_run(r: run_trees.RunTree) -> None: with rh.tracing_context(enabled=True): try: args = ( - (example.inputs, example.attachment_urls) + (example.inputs, example.attachments) if include_attachments else (example.inputs,) ) @@ -1044,9 +1044,9 @@ def _get_run(r: run_trees.RunTree) -> None: client=client, ), ) - if include_attachments and example.attachment_urls is not None: - for attachment in example.attachment_urls: - reader = example.attachment_urls[attachment]["reader"] + if include_attachments and example.attachments is not None: + for attachment in example.attachments: + reader = example.attachments[attachment]["reader"] reader.seek(0) except Exception as e: logger.error( diff --git a/python/langsmith/evaluation/_runner.py b/python/langsmith/evaluation/_runner.py index 199d8fa22..5a00585d5 100644 --- a/python/langsmith/evaluation/_runner.py +++ b/python/langsmith/evaluation/_runner.py @@ -1834,7 +1834,7 @@ def _get_run(r: rt.RunTree) -> None: ) try: args = ( - (example.inputs, example.attachment_urls) + (example.inputs, example.attachments) if include_attachments else (example.inputs,) ) @@ -1842,9 +1842,9 @@ def _get_run(r: rt.RunTree) -> None: *args, langsmith_extra=langsmith_extra, ) - if include_attachments and example.attachment_urls is not None: - for attachment in example.attachment_urls: - reader = example.attachment_urls[attachment]["reader"] + if include_attachments and example.attachments is not None: + for attachment in example.attachments: + reader = example.attachments[attachment]["reader"] reader.seek(0) except Exception as e: logger.error( diff --git a/python/langsmith/schemas.py b/python/langsmith/schemas.py index 41fa76cb0..5b226a830 100644 --- a/python/langsmith/schemas.py +++ b/python/langsmith/schemas.py @@ -146,7 +146,7 @@ class Example(ExampleBase): modified_at: Optional[datetime] = Field(default=None) runs: List[Run] = Field(default_factory=list) source_run_id: Optional[UUID] = None - attachment_urls: Optional[Dict[str, AttachmentInfo]] = Field(default=None) + attachments: Optional[Dict[str, AttachmentInfo]] = Field(default=None) """Dictionary with attachment names as keys and a tuple of the S3 url and a reader of the data for the file.""" _host_url: Optional[str] = PrivateAttr(default=None) From 2b385b631ab32348fd00123993bc38a6438f49ee Mon Sep 17 00:00:00 2001 From: isaac hershenson Date: Mon, 9 Dec 2024 14:20:06 -0800 Subject: [PATCH 3/3] x --- python/tests/integration_tests/test_client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/tests/integration_tests/test_client.py b/python/tests/integration_tests/test_client.py index ae866521c..83efc397e 100644 --- a/python/tests/integration_tests/test_client.py +++ b/python/tests/integration_tests/test_client.py @@ -439,7 +439,7 @@ def test_upload_examples_multipart(langchain_client: Client): # Verify example with ID was created with correct ID example_with_id = [ex for ex in examples if ex.id == example_id][0] assert example_with_id.inputs["text"] == "hello world" - assert "test_file" in example_with_id.attachment_urls + assert "test_file" in example_with_id.attachments # Verify example with outputs and multiple attachments example_with_outputs = next( @@ -447,9 +447,9 @@ def test_upload_examples_multipart(langchain_client: Client): for ex in examples if ex.outputs and ex.outputs.get("response") == "test response" ) - assert len(example_with_outputs.attachment_urls) == 2 - assert "file1" in example_with_outputs.attachment_urls - assert "file2" in example_with_outputs.attachment_urls + assert len(example_with_outputs.attachments) == 2 + assert "file1" in example_with_outputs.attachments + assert "file2" in example_with_outputs.attachments # Test uploading to non-existent dataset fails fake_id = uuid4()