From 4c70ffff0145b7d2468708f5ee58d5e7db000ea7 Mon Sep 17 00:00:00 2001 From: Erick Friis Date: Mon, 9 Dec 2024 10:02:55 -0800 Subject: [PATCH] standard-tests: sync/async vectorstore tests conditional (#28636) Co-authored-by: Chester Curme --- .../integration_tests/vectorstores.py | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/libs/standard-tests/langchain_tests/integration_tests/vectorstores.py b/libs/standard-tests/langchain_tests/integration_tests/vectorstores.py index c043023f3ac48..6b8d8d1d565d6 100644 --- a/libs/standard-tests/langchain_tests/integration_tests/vectorstores.py +++ b/libs/standard-tests/langchain_tests/integration_tests/vectorstores.py @@ -76,6 +76,21 @@ def vectorstore(self) -> Generator[VectorStore, None, None]: # type: ignore store.delete_collection() pass + Note that by default we enable both sync and async tests. To disable either, + override the ``has_sync`` or ``has_async`` properties to ``False`` in the + subclass. For example: + + .. code-block:: python + + class TestParrotVectorStore(VectorStoreIntegrationTests): + @pytest.fixture() + def vectorstore(self) -> Generator[VectorStore, None, None]: # type: ignore + ... + + @property + def has_async(self) -> bool: + return False + .. note:: API references for individual test methods include troubleshooting tips. """ # noqa: E501 @@ -88,6 +103,20 @@ def vectorstore(self) -> VectorStore: The returned vectorstore should be EMPTY. """ + @property + def has_sync(self) -> bool: + """ + Configurable property to enable or disable sync tests. + """ + return True + + @property + def has_async(self) -> bool: + """ + Configurable property to enable or disable async tests. + """ + return True + @staticmethod def get_embeddings() -> Embeddings: """A pre-defined embeddings model that should be used for this test. @@ -110,6 +139,9 @@ def test_vectorstore_is_empty(self, vectorstore: VectorStore) -> None: ``VectorStoreIntegrationTests``) initializes an empty vector store in the ``vectorestore`` fixture. """ + if not self.has_sync: + pytest.skip("Sync tests not supported.") + assert vectorstore.similarity_search("foo", k=1) == [] def test_add_documents(self, vectorstore: VectorStore) -> None: @@ -123,6 +155,9 @@ def test_add_documents(self, vectorstore: VectorStore) -> None: 2. Calling ``.similarity_search`` for the top ``k`` similar documents does not threshold by score. 3. We do not mutate the original document object when adding it to the vector store (e.g., by adding an ID). """ # noqa: E501 + if not self.has_sync: + pytest.skip("Sync tests not supported.") + original_documents = [ Document(page_content="foo", metadata={"id": 1}), Document(page_content="bar", metadata={"id": 2}), @@ -152,6 +187,9 @@ def test_vectorstore_still_empty(self, vectorstore: VectorStore) -> None: ``VectorStoreIntegrationTests``) correctly clears the vector store in the ``finally`` block. """ + if not self.has_sync: + pytest.skip("Sync tests not supported.") + assert vectorstore.similarity_search("foo", k=1) == [] def test_deleting_documents(self, vectorstore: VectorStore) -> None: @@ -163,6 +201,9 @@ def test_deleting_documents(self, vectorstore: VectorStore) -> None: passed in through ``ids``, and that ``delete`` correctly removes documents. """ + if not self.has_sync: + pytest.skip("Sync tests not supported.") + documents = [ Document(page_content="foo", metadata={"id": 1}), Document(page_content="bar", metadata={"id": 2}), @@ -181,6 +222,9 @@ def test_deleting_bulk_documents(self, vectorstore: VectorStore) -> None: If this test fails, check that ``delete`` correctly removes multiple documents when given a list of IDs. """ + if not self.has_sync: + pytest.skip("Sync tests not supported.") + documents = [ Document(page_content="foo", metadata={"id": 1}), Document(page_content="bar", metadata={"id": 2}), @@ -200,6 +244,9 @@ def test_delete_missing_content(self, vectorstore: VectorStore) -> None: If this test fails, check that ``delete`` does not raise an exception when deleting IDs that do not exist. """ + if not self.has_sync: + pytest.skip("Sync tests not supported.") + vectorstore.delete(["1"]) vectorstore.delete(["1", "2", "3"]) @@ -214,6 +261,9 @@ def test_add_documents_with_ids_is_idempotent( same IDs has the same effect as adding it once (i.e., it does not duplicate the documents). """ + if not self.has_sync: + pytest.skip("Sync tests not supported.") + documents = [ Document(page_content="foo", metadata={"id": 1}), Document(page_content="bar", metadata={"id": 2}), @@ -235,6 +285,9 @@ def test_add_documents_by_id_with_mutation(self, vectorstore: VectorStore) -> No ID that already exists in the vector store, the content is updated rather than duplicated. """ + if not self.has_sync: + pytest.skip("Sync tests not supported.") + documents = [ Document(page_content="foo", metadata={"id": 1}), Document(page_content="bar", metadata={"id": 2}), @@ -283,6 +336,9 @@ def test_get_by_ids(self, vectorstore: VectorStore) -> None: def test_get_by_ids(self, vectorstore: VectorStore) -> None: super().test_get_by_ids(vectorstore) """ + if not self.has_sync: + pytest.skip("Sync tests not supported.") + documents = [ Document(page_content="foo", metadata={"id": 1}), Document(page_content="bar", metadata={"id": 2}), @@ -313,6 +369,9 @@ def test_get_by_ids_missing(self, vectorstore: VectorStore) -> None: def test_get_by_ids_missing(self, vectorstore: VectorStore) -> None: super().test_get_by_ids_missing(vectorstore) """ # noqa: E501 + if not self.has_sync: + pytest.skip("Sync tests not supported.") + # This should not raise an exception documents = vectorstore.get_by_ids(["1", "2", "3"]) assert documents == [] @@ -339,6 +398,9 @@ def test_add_documents_documents(self, vectorstore: VectorStore) -> None: def test_add_documents_documents(self, vectorstore: VectorStore) -> None: super().test_add_documents_documents(vectorstore) """ # noqa: E501 + if not self.has_sync: + pytest.skip("Sync tests not supported.") + documents = [ Document(page_content="foo", metadata={"id": 1}), Document(page_content="bar", metadata={"id": 2}), @@ -373,6 +435,9 @@ def test_add_documents_with_existing_ids(self, vectorstore: VectorStore) -> None def test_add_documents_with_existing_ids(self, vectorstore: VectorStore) -> None: super().test_add_documents_with_existing_ids(vectorstore) """ # noqa: E501 + if not self.has_sync: + pytest.skip("Sync tests not supported.") + documents = [ Document(id="foo", page_content="foo", metadata={"id": 1}), Document(page_content="bar", metadata={"id": 2}), @@ -393,6 +458,9 @@ async def test_vectorstore_is_empty_async(self, vectorstore: VectorStore) -> Non ``VectorStoreIntegrationTests``) initializes an empty vector store in the ``vectorestore`` fixture. """ + if not self.has_async: + pytest.skip("Async tests not supported.") + assert await vectorstore.asimilarity_search("foo", k=1) == [] async def test_add_documents_async(self, vectorstore: VectorStore) -> None: @@ -406,6 +474,9 @@ async def test_add_documents_async(self, vectorstore: VectorStore) -> None: 2. Calling ``.asimilarity_search`` for the top ``k`` similar documents does not threshold by score. 3. We do not mutate the original document object when adding it to the vector store (e.g., by adding an ID). """ # noqa: E501 + if not self.has_async: + pytest.skip("Async tests not supported.") + original_documents = [ Document(page_content="foo", metadata={"id": 1}), Document(page_content="bar", metadata={"id": 2}), @@ -438,6 +509,9 @@ async def test_vectorstore_still_empty_async( ``VectorStoreIntegrationTests``) correctly clears the vector store in the ``finally`` block. """ + if not self.has_async: + pytest.skip("Async tests not supported.") + assert await vectorstore.asimilarity_search("foo", k=1) == [] async def test_deleting_documents_async(self, vectorstore: VectorStore) -> None: @@ -449,6 +523,9 @@ async def test_deleting_documents_async(self, vectorstore: VectorStore) -> None: passed in through ``ids``, and that ``delete`` correctly removes documents. """ + if not self.has_async: + pytest.skip("Async tests not supported.") + documents = [ Document(page_content="foo", metadata={"id": 1}), Document(page_content="bar", metadata={"id": 2}), @@ -469,6 +546,9 @@ async def test_deleting_bulk_documents_async( If this test fails, check that ``adelete`` correctly removes multiple documents when given a list of IDs. """ + if not self.has_async: + pytest.skip("Async tests not supported.") + documents = [ Document(page_content="foo", metadata={"id": 1}), Document(page_content="bar", metadata={"id": 2}), @@ -488,6 +568,9 @@ async def test_delete_missing_content_async(self, vectorstore: VectorStore) -> N If this test fails, check that ``adelete`` does not raise an exception when deleting IDs that do not exist. """ + if not self.has_async: + pytest.skip("Async tests not supported.") + await vectorstore.adelete(["1"]) await vectorstore.adelete(["1", "2", "3"]) @@ -502,6 +585,9 @@ async def test_add_documents_with_ids_is_idempotent_async( same IDs has the same effect as adding it once (i.e., it does not duplicate the documents). """ + if not self.has_async: + pytest.skip("Async tests not supported.") + documents = [ Document(page_content="foo", metadata={"id": 1}), Document(page_content="bar", metadata={"id": 2}), @@ -525,6 +611,9 @@ async def test_add_documents_by_id_with_mutation_async( ID that already exists in the vector store, the content is updated rather than duplicated. """ + if not self.has_async: + pytest.skip("Async tests not supported.") + documents = [ Document(page_content="foo", metadata={"id": 1}), Document(page_content="bar", metadata={"id": 2}), @@ -573,6 +662,9 @@ async def test_get_by_ids_async(self, vectorstore: VectorStore) -> None: async def test_get_by_ids(self, vectorstore: VectorStore) -> None: await super().test_get_by_ids(vectorstore) """ + if not self.has_async: + pytest.skip("Async tests not supported.") + documents = [ Document(page_content="foo", metadata={"id": 1}), Document(page_content="bar", metadata={"id": 2}), @@ -603,6 +695,9 @@ async def test_get_by_ids_missing_async(self, vectorstore: VectorStore) -> None: async def test_get_by_ids_missing(self, vectorstore: VectorStore) -> None: await super().test_get_by_ids_missing(vectorstore) """ # noqa: E501 + if not self.has_async: + pytest.skip("Async tests not supported.") + # This should not raise an exception assert await vectorstore.aget_by_ids(["1", "2", "3"]) == [] @@ -630,6 +725,9 @@ async def test_add_documents_documents_async( async def test_add_documents_documents(self, vectorstore: VectorStore) -> None: await super().test_add_documents_documents(vectorstore) """ # noqa: E501 + if not self.has_async: + pytest.skip("Async tests not supported.") + documents = [ Document(page_content="foo", metadata={"id": 1}), Document(page_content="bar", metadata={"id": 2}), @@ -666,6 +764,9 @@ async def test_add_documents_with_existing_ids_async( async def test_add_documents_with_existing_ids(self, vectorstore: VectorStore) -> None: await super().test_add_documents_with_existing_ids(vectorstore) """ # noqa: E501 + if not self.has_async: + pytest.skip("Async tests not supported.") + documents = [ Document(id="foo", page_content="foo", metadata={"id": 1}), Document(page_content="bar", metadata={"id": 2}),