From 47d034a16807cd23b7a67e4eada0555478330b8b Mon Sep 17 00:00:00 2001 From: Vladimir Blagojevic Date: Thu, 15 Aug 2024 11:03:37 +0100 Subject: [PATCH 01/25] Update qdrant test_run_filters (#969) --- integrations/qdrant/tests/test_retriever.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/integrations/qdrant/tests/test_retriever.py b/integrations/qdrant/tests/test_retriever.py index a92f6917f..5a558db3b 100644 --- a/integrations/qdrant/tests/test_retriever.py +++ b/integrations/qdrant/tests/test_retriever.py @@ -151,7 +151,10 @@ def test_run_filters(self, filterable_docs: List[Document]): filters={"field": "meta.chapter", "operator": "==", "value": "abstract"}, return_embedding=False, )["documents"] - assert len(results) == 3 + # we need to combine init filter and run filter as the policy is MERGE + # when we combine these filters we use AND logical operator by default + # so the result should be 1 as we have only one document that matches both filters + assert len(results) == 1 for document in results: assert document.embedding is None From ddfd74b62a257ed041284e9f5d7ebecc56ee35eb Mon Sep 17 00:00:00 2001 From: Vladimir Blagojevic Date: Fri, 16 Aug 2024 11:34:43 +0100 Subject: [PATCH 02/25] Ruff update, don't ruff tests (#982) --- integrations/jina/pyproject.toml | 4 ++-- integrations/jina/tests/test_document_embedder.py | 1 + integrations/jina/tests/test_ranker.py | 1 + integrations/jina/tests/test_text_embedder.py | 1 + 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/integrations/jina/pyproject.toml b/integrations/jina/pyproject.toml index fa2fd50ed..908633686 100644 --- a/integrations/jina/pyproject.toml +++ b/integrations/jina/pyproject.toml @@ -62,8 +62,8 @@ detached = true dependencies = ["black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243"] [tool.hatch.envs.lint.scripts] typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}" -style = ["ruff check {args:.}", "black --check --diff {args:.}"] -fmt = ["black {args:.}", "ruff --fix {args:.}", "style"] +style = ["ruff check {args:. --exclude tests/}", "black --check --diff {args:.}"] +fmt = ["black {args:.}", "ruff --fix {args:. --exclude tests/}", "style"] all = ["style", "typing"] [tool.black] diff --git a/integrations/jina/tests/test_document_embedder.py b/integrations/jina/tests/test_document_embedder.py index a9ba23ec0..9d63f8302 100644 --- a/integrations/jina/tests/test_document_embedder.py +++ b/integrations/jina/tests/test_document_embedder.py @@ -8,6 +8,7 @@ import requests from haystack import Document from haystack.utils import Secret + from haystack_integrations.components.embedders.jina import JinaDocumentEmbedder diff --git a/integrations/jina/tests/test_ranker.py b/integrations/jina/tests/test_ranker.py index 7ece6806d..e7fdaade0 100644 --- a/integrations/jina/tests/test_ranker.py +++ b/integrations/jina/tests/test_ranker.py @@ -8,6 +8,7 @@ import requests from haystack import Document from haystack.utils import Secret + from haystack_integrations.components.rankers.jina import JinaRanker diff --git a/integrations/jina/tests/test_text_embedder.py b/integrations/jina/tests/test_text_embedder.py index 7cb669c68..5c0f80d02 100644 --- a/integrations/jina/tests/test_text_embedder.py +++ b/integrations/jina/tests/test_text_embedder.py @@ -7,6 +7,7 @@ import pytest import requests from haystack.utils import Secret + from haystack_integrations.components.embedders.jina import JinaTextEmbedder From ac2680c2856c1c8c0591ade66a989f3abd4b3545 Mon Sep 17 00:00:00 2001 From: Vladimir Blagojevic Date: Fri, 16 Aug 2024 11:35:27 +0100 Subject: [PATCH 03/25] chore: Chroma - ruff update, don't ruff tests (#983) * Ruff update, don't ruff tests * More update to pass CI --- integrations/chroma/example/example.py | 8 +++++--- integrations/chroma/pyproject.toml | 13 +++++++------ .../components/retrievers/chroma/retriever.py | 1 + integrations/chroma/tests/test_document_store.py | 1 + integrations/chroma/tests/test_retriever.py | 1 + 5 files changed, 15 insertions(+), 9 deletions(-) diff --git a/integrations/chroma/example/example.py b/integrations/chroma/example/example.py index d9b0f414e..3adfc4183 100644 --- a/integrations/chroma/example/example.py +++ b/integrations/chroma/example/example.py @@ -1,4 +1,5 @@ # Colab: https://colab.research.google.com/drive/1YpDetI8BRbObPDEVdfqUcwhEX9UUXP-m?usp=sharing + import os from pathlib import Path @@ -6,14 +7,15 @@ from haystack.components.converters import TextFileToDocument from haystack.components.writers import DocumentWriter -from haystack_integrations.document_stores.chroma import ChromaDocumentStore from haystack_integrations.components.retrievers.chroma import ChromaQueryTextRetriever +from haystack_integrations.document_stores.chroma import ChromaDocumentStore + +# Chroma is used in-memory so we use the same instances in the two pipelines below +document_store = ChromaDocumentStore() HERE = Path(__file__).resolve().parent file_paths = [HERE / "data" / Path(name) for name in os.listdir("data")] -# Chroma is used in-memory so we use the same instances in the two pipelines below -document_store = ChromaDocumentStore() indexing = Pipeline() indexing.add_component("converter", TextFileToDocument()) diff --git a/integrations/chroma/pyproject.toml b/integrations/chroma/pyproject.toml index b4591e041..cebfa1b9d 100644 --- a/integrations/chroma/pyproject.toml +++ b/integrations/chroma/pyproject.toml @@ -70,8 +70,8 @@ dependencies = [ ] [tool.hatch.envs.lint.scripts] typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}" -style = ["ruff check {args:.}", "black --check --diff {args:.}"] -fmt = ["black {args:.}", "ruff --fix {args:.}", "style"] +style = ["ruff check {args:. --exclude tests/}", "black --check --diff {args:.}"] +fmt = ["black {args:.}", "ruff --fix {args:. --exclude tests/}", "style"] all = ["style", "typing"] [tool.hatch.metadata] @@ -85,7 +85,7 @@ skip-string-normalization = true [tool.ruff] target-version = "py38" line-length = 120 -select = [ +lint.select = [ "A", "ARG", "B", @@ -112,7 +112,7 @@ select = [ "W", "YTT", ] -ignore = [ +lint.ignore = [ # Allow non-abstract empty methods in abstract base classes "B027", # Allow boolean positional values in function calls, like `dict.get(... True)` @@ -130,14 +130,14 @@ ignore = [ # Ignore unused params "ARG002", ] -unfixable = [ +lint.unfixable = [ # Don't touch unused imports "F401", ] exclude = ["example"] [tool.ruff.isort] -known-first-party = ["src"] +known-first-party = ["src", "example", "tests"] [tool.ruff.flake8-tidy-imports] ban-relative-imports = "parents" @@ -145,6 +145,7 @@ ban-relative-imports = "parents" [tool.ruff.per-file-ignores] # Tests can use magic values, assertions, and relative imports "tests/**/*" = ["PLR2004", "S101", "TID252"] +"example/**/*" = ["T201"] [tool.coverage.run] source = ["haystack_integrations"] diff --git a/integrations/chroma/src/haystack_integrations/components/retrievers/chroma/retriever.py b/integrations/chroma/src/haystack_integrations/components/retrievers/chroma/retriever.py index 71ac3457e..a19022e23 100644 --- a/integrations/chroma/src/haystack_integrations/components/retrievers/chroma/retriever.py +++ b/integrations/chroma/src/haystack_integrations/components/retrievers/chroma/retriever.py @@ -6,6 +6,7 @@ from haystack import Document, component, default_from_dict, default_to_dict from haystack.document_stores.types import FilterPolicy from haystack.document_stores.types.filter_policy import apply_filter_policy + from haystack_integrations.document_stores.chroma import ChromaDocumentStore diff --git a/integrations/chroma/tests/test_document_store.py b/integrations/chroma/tests/test_document_store.py index b05c9ccfc..d4b6ed272 100644 --- a/integrations/chroma/tests/test_document_store.py +++ b/integrations/chroma/tests/test_document_store.py @@ -16,6 +16,7 @@ DeleteDocumentsTest, LegacyFilterDocumentsTest, ) + from haystack_integrations.document_stores.chroma import ChromaDocumentStore diff --git a/integrations/chroma/tests/test_retriever.py b/integrations/chroma/tests/test_retriever.py index f0e71828d..99a4c34e6 100644 --- a/integrations/chroma/tests/test_retriever.py +++ b/integrations/chroma/tests/test_retriever.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 import pytest from haystack.document_stores.types import FilterPolicy + from haystack_integrations.components.retrievers.chroma import ChromaQueryTextRetriever from haystack_integrations.document_stores.chroma import ChromaDocumentStore From d77c4ea65499e9e5fa9b5f5d01141c559309cf9d Mon Sep 17 00:00:00 2001 From: Vladimir Blagojevic Date: Fri, 16 Aug 2024 11:38:34 +0100 Subject: [PATCH 04/25] Ruff update, don't ruff tests (#992) --- integrations/langfuse/pyproject.toml | 4 ++-- .../components/connectors/langfuse/langfuse_connector.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/integrations/langfuse/pyproject.toml b/integrations/langfuse/pyproject.toml index cf7b85b64..d92c62668 100644 --- a/integrations/langfuse/pyproject.toml +++ b/integrations/langfuse/pyproject.toml @@ -67,8 +67,8 @@ dependencies = ["black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243"] [tool.hatch.envs.lint.scripts] typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}" -style = ["ruff check {args:.}", "black --check --diff {args:.}"] -fmt = ["black {args:.}", "ruff --fix {args:.}", "style"] +style = ["ruff check {args:. --exclude tests/}", "black --check --diff {args:.}"] +fmt = ["black {args:.}", "ruff --fix {args:. --exclude tests/}", "style"] all = ["style", "typing"] [tool.hatch.metadata] diff --git a/integrations/langfuse/src/haystack_integrations/components/connectors/langfuse/langfuse_connector.py b/integrations/langfuse/src/haystack_integrations/components/connectors/langfuse/langfuse_connector.py index 51703823e..ab93c3433 100644 --- a/integrations/langfuse/src/haystack_integrations/components/connectors/langfuse/langfuse_connector.py +++ b/integrations/langfuse/src/haystack_integrations/components/connectors/langfuse/langfuse_connector.py @@ -1,6 +1,6 @@ from haystack import component, tracing -from haystack_integrations.tracing.langfuse import LangfuseTracer +from haystack_integrations.tracing.langfuse import LangfuseTracer from langfuse import Langfuse From 1f72e124336187c18dd154b25b0cfe5ddc2e7b3f Mon Sep 17 00:00:00 2001 From: Vladimir Blagojevic Date: Fri, 16 Aug 2024 11:39:13 +0100 Subject: [PATCH 05/25] chore: Update mongodb test for the new `apply_filter_policy` usage (#971) * Use new filters, test merge on the same field use case * Update all tests to use new filter syntax * Black it --- .../mongodb_atlas/tests/test_retriever.py | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/integrations/mongodb_atlas/tests/test_retriever.py b/integrations/mongodb_atlas/tests/test_retriever.py index 56eec928f..c25e7487f 100644 --- a/integrations/mongodb_atlas/tests/test_retriever.py +++ b/integrations/mongodb_atlas/tests/test_retriever.py @@ -12,7 +12,6 @@ class TestRetriever: - @pytest.fixture def mock_client(self): with patch( @@ -44,11 +43,11 @@ def test_init(self): mock_store = Mock(spec=MongoDBAtlasDocumentStore) retriever = MongoDBAtlasEmbeddingRetriever( document_store=mock_store, - filters={"field": "value"}, + filters={"field": "meta.some_field", "operator": "==", "value": "SomeValue"}, top_k=5, ) assert retriever.document_store == mock_store - assert retriever.filters == {"field": "value"} + assert retriever.filters == {"field": "meta.some_field", "operator": "==", "value": "SomeValue"} assert retriever.top_k == 5 assert retriever.filter_policy == FilterPolicy.REPLACE @@ -56,12 +55,12 @@ def test_init_filter_policy_merge(self): mock_store = Mock(spec=MongoDBAtlasDocumentStore) retriever = MongoDBAtlasEmbeddingRetriever( document_store=mock_store, - filters={"field": "value"}, + filters={"field": "meta.some_field", "operator": "==", "value": "SomeValue"}, top_k=5, filter_policy=FilterPolicy.MERGE, ) assert retriever.document_store == mock_store - assert retriever.filters == {"field": "value"} + assert retriever.filters == {"field": "meta.some_field", "operator": "==", "value": "SomeValue"} assert retriever.top_k == 5 assert retriever.filter_policy == FilterPolicy.MERGE @@ -189,12 +188,18 @@ def test_run_merge_policy_filter(self): mock_store._embedding_retrieval.return_value = [doc] retriever = MongoDBAtlasEmbeddingRetriever( - document_store=mock_store, filters={"foo": "boo"}, filter_policy=FilterPolicy.MERGE + document_store=mock_store, + filters={"field": "meta.some_field", "operator": "==", "value": "SomeValue"}, + filter_policy=FilterPolicy.MERGE, ) - res = retriever.run(query_embedding=[0.3, 0.5], filters={"field": "value"}) - + res = retriever.run( + query_embedding=[0.3, 0.5], filters={"field": "meta.some_field", "operator": "==", "value": "Test"} + ) + # as the both init and run filters are filtering the same field, the run filter takes precedence mock_store._embedding_retrieval.assert_called_once_with( - query_embedding=[0.3, 0.5], filters={"field": "value", "foo": "boo"}, top_k=10 + query_embedding=[0.3, 0.5], + filters={"field": "meta.some_field", "operator": "==", "value": "Test"}, + top_k=10, ) assert res == {"documents": [doc]} From e6721ea7ff23bceaa9ecb7219d87cef5156dd33e Mon Sep 17 00:00:00 2001 From: Vladimir Blagojevic Date: Fri, 16 Aug 2024 11:39:50 +0100 Subject: [PATCH 06/25] Ruff update, don't ruff tests (#986) --- integrations/weaviate/pyproject.toml | 4 ++-- .../components/retrievers/weaviate/bm25_retriever.py | 1 + .../retrievers/weaviate/embedding_retriever.py | 1 + integrations/weaviate/tests/test_auth.py | 9 +++++---- integrations/weaviate/tests/test_bm25_retriever.py | 1 + integrations/weaviate/tests/test_document_store.py | 11 ++++++----- .../weaviate/tests/test_embedding_retriever.py | 1 + 7 files changed, 17 insertions(+), 11 deletions(-) diff --git a/integrations/weaviate/pyproject.toml b/integrations/weaviate/pyproject.toml index 14b60fe12..4f9a3245a 100644 --- a/integrations/weaviate/pyproject.toml +++ b/integrations/weaviate/pyproject.toml @@ -65,8 +65,8 @@ detached = true dependencies = ["black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243"] [tool.hatch.envs.lint.scripts] typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}" -style = ["ruff check {args:.}", "black --check --diff {args:.}"] -fmt = ["black {args:.}", "ruff --fix {args:.}", "style"] +style = ["ruff check {args:. --exclude tests/}", "black --check --diff {args:.}"] +fmt = ["black {args:.}", "ruff --fix {args:. --exclude tests/}", "style"] all = ["style", "typing"] [tool.black] diff --git a/integrations/weaviate/src/haystack_integrations/components/retrievers/weaviate/bm25_retriever.py b/integrations/weaviate/src/haystack_integrations/components/retrievers/weaviate/bm25_retriever.py index fec0b81e6..a238647f2 100644 --- a/integrations/weaviate/src/haystack_integrations/components/retrievers/weaviate/bm25_retriever.py +++ b/integrations/weaviate/src/haystack_integrations/components/retrievers/weaviate/bm25_retriever.py @@ -7,6 +7,7 @@ from haystack import Document, component, default_from_dict, default_to_dict from haystack.document_stores.types import FilterPolicy from haystack.document_stores.types.filter_policy import apply_filter_policy + from haystack_integrations.document_stores.weaviate import WeaviateDocumentStore diff --git a/integrations/weaviate/src/haystack_integrations/components/retrievers/weaviate/embedding_retriever.py b/integrations/weaviate/src/haystack_integrations/components/retrievers/weaviate/embedding_retriever.py index 8688b4145..3a0f51e47 100644 --- a/integrations/weaviate/src/haystack_integrations/components/retrievers/weaviate/embedding_retriever.py +++ b/integrations/weaviate/src/haystack_integrations/components/retrievers/weaviate/embedding_retriever.py @@ -7,6 +7,7 @@ from haystack import Document, component, default_from_dict, default_to_dict from haystack.document_stores.types import FilterPolicy from haystack.document_stores.types.filter_policy import apply_filter_policy + from haystack_integrations.document_stores.weaviate import WeaviateDocumentStore diff --git a/integrations/weaviate/tests/test_auth.py b/integrations/weaviate/tests/test_auth.py index 3ad75e206..8beaf1f85 100644 --- a/integrations/weaviate/tests/test_auth.py +++ b/integrations/weaviate/tests/test_auth.py @@ -2,6 +2,11 @@ # # SPDX-License-Identifier: Apache-2.0 +from weaviate.auth import AuthApiKey as WeaviateAuthApiKey +from weaviate.auth import AuthBearerToken as WeaviateAuthBearerToken +from weaviate.auth import AuthClientCredentials as WeaviateAuthClientCredentials +from weaviate.auth import AuthClientPassword as WeaviateAuthClientPassword + from haystack_integrations.document_stores.weaviate.auth import ( AuthApiKey, AuthBearerToken, @@ -9,10 +14,6 @@ AuthClientPassword, AuthCredentials, ) -from weaviate.auth import AuthApiKey as WeaviateAuthApiKey -from weaviate.auth import AuthBearerToken as WeaviateAuthBearerToken -from weaviate.auth import AuthClientCredentials as WeaviateAuthClientCredentials -from weaviate.auth import AuthClientPassword as WeaviateAuthClientPassword class TestAuthApiKey: diff --git a/integrations/weaviate/tests/test_bm25_retriever.py b/integrations/weaviate/tests/test_bm25_retriever.py index 3720daa85..bc28775cc 100644 --- a/integrations/weaviate/tests/test_bm25_retriever.py +++ b/integrations/weaviate/tests/test_bm25_retriever.py @@ -6,6 +6,7 @@ import pytest from haystack.document_stores.types import FilterPolicy + from haystack_integrations.components.retrievers.weaviate import WeaviateBM25Retriever from haystack_integrations.document_stores.weaviate import WeaviateDocumentStore diff --git a/integrations/weaviate/tests/test_document_store.py b/integrations/weaviate/tests/test_document_store.py index 068212686..8d531cade 100644 --- a/integrations/weaviate/tests/test_document_store.py +++ b/integrations/weaviate/tests/test_document_store.py @@ -22,11 +22,6 @@ WriteDocumentsTest, ) from haystack.utils.auth import Secret -from haystack_integrations.document_stores.weaviate.auth import AuthApiKey -from haystack_integrations.document_stores.weaviate.document_store import ( - DOCUMENT_COLLECTION_PROPERTIES, - WeaviateDocumentStore, -) from numpy import array as np_array from numpy import array_equal as np_array_equal from numpy import float32 as np_float32 @@ -41,6 +36,12 @@ EmbeddedOptions, ) +from haystack_integrations.document_stores.weaviate.auth import AuthApiKey +from haystack_integrations.document_stores.weaviate.document_store import ( + DOCUMENT_COLLECTION_PROPERTIES, + WeaviateDocumentStore, +) + @patch("haystack_integrations.document_stores.weaviate.document_store.weaviate.WeaviateClient") def test_init_is_lazy(_mock_client): diff --git a/integrations/weaviate/tests/test_embedding_retriever.py b/integrations/weaviate/tests/test_embedding_retriever.py index 13f214dd1..75626e7c3 100644 --- a/integrations/weaviate/tests/test_embedding_retriever.py +++ b/integrations/weaviate/tests/test_embedding_retriever.py @@ -6,6 +6,7 @@ import pytest from haystack.document_stores.types import FilterPolicy + from haystack_integrations.components.retrievers.weaviate import WeaviateEmbeddingRetriever from haystack_integrations.document_stores.weaviate import WeaviateDocumentStore From c606555257c01d16e50a65e11dd544447401a5b9 Mon Sep 17 00:00:00 2001 From: Vladimir Blagojevic Date: Fri, 16 Aug 2024 11:40:16 +0100 Subject: [PATCH 07/25] Ruff update, don't ruff tests (#985) --- integrations/ollama/examples/chat_generator_example.py | 1 + integrations/ollama/examples/embedders_example.py | 1 + integrations/ollama/examples/generator_example.py | 1 + integrations/ollama/pyproject.toml | 4 ++-- integrations/ollama/tests/test_chat_generator.py | 3 ++- integrations/ollama/tests/test_document_embedder.py | 3 ++- integrations/ollama/tests/test_generator.py | 3 ++- integrations/ollama/tests/test_text_embedder.py | 3 ++- 8 files changed, 13 insertions(+), 6 deletions(-) diff --git a/integrations/ollama/examples/chat_generator_example.py b/integrations/ollama/examples/chat_generator_example.py index 065a6a1b4..834df78fb 100644 --- a/integrations/ollama/examples/chat_generator_example.py +++ b/integrations/ollama/examples/chat_generator_example.py @@ -6,6 +6,7 @@ # docker exec ollama ollama pull orca-mini from haystack.dataclasses import ChatMessage + from haystack_integrations.components.generators.ollama import OllamaChatGenerator messages = [ diff --git a/integrations/ollama/examples/embedders_example.py b/integrations/ollama/examples/embedders_example.py index 21a36e2a2..84e0d3c3e 100644 --- a/integrations/ollama/examples/embedders_example.py +++ b/integrations/ollama/examples/embedders_example.py @@ -7,6 +7,7 @@ from haystack import Document, Pipeline from haystack.components.retrievers.in_memory import InMemoryEmbeddingRetriever from haystack.document_stores.in_memory import InMemoryDocumentStore + from haystack_integrations.components.embedders.ollama.document_embedder import OllamaDocumentEmbedder from haystack_integrations.components.embedders.ollama.text_embedder import OllamaTextEmbedder diff --git a/integrations/ollama/examples/generator_example.py b/integrations/ollama/examples/generator_example.py index b674e676b..e438f2765 100644 --- a/integrations/ollama/examples/generator_example.py +++ b/integrations/ollama/examples/generator_example.py @@ -9,6 +9,7 @@ from haystack.components.builders.prompt_builder import PromptBuilder from haystack.components.retrievers import InMemoryBM25Retriever from haystack.document_stores.in_memory import InMemoryDocumentStore + from haystack_integrations.components.generators.ollama import OllamaGenerator document_store = InMemoryDocumentStore() diff --git a/integrations/ollama/pyproject.toml b/integrations/ollama/pyproject.toml index 5187de31f..57aee153b 100644 --- a/integrations/ollama/pyproject.toml +++ b/integrations/ollama/pyproject.toml @@ -71,8 +71,8 @@ dependencies = ["black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243"] [tool.hatch.envs.lint.scripts] typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}" -style = ["ruff check {args:.}", "black --check --diff {args:.}"] -fmt = ["black {args:.}", "ruff --fix {args:.}", "style"] +style = ["ruff check {args:. --exclude tests/, examples/ }", "black --check --diff {args:.}"] +fmt = ["black {args:.}", "ruff --fix {args:. --exclude tests/, examples/ }", "style"] all = ["style", "typing"] [tool.hatch.metadata] diff --git a/integrations/ollama/tests/test_chat_generator.py b/integrations/ollama/tests/test_chat_generator.py index e09208bb8..dd4e746aa 100644 --- a/integrations/ollama/tests/test_chat_generator.py +++ b/integrations/ollama/tests/test_chat_generator.py @@ -3,9 +3,10 @@ import pytest from haystack.dataclasses import ChatMessage, ChatRole -from haystack_integrations.components.generators.ollama import OllamaChatGenerator from requests import HTTPError, Response +from haystack_integrations.components.generators.ollama import OllamaChatGenerator + @pytest.fixture def chat_messages() -> List[ChatMessage]: diff --git a/integrations/ollama/tests/test_document_embedder.py b/integrations/ollama/tests/test_document_embedder.py index 012ad9eae..0f5b55881 100644 --- a/integrations/ollama/tests/test_document_embedder.py +++ b/integrations/ollama/tests/test_document_embedder.py @@ -1,8 +1,9 @@ import pytest from haystack import Document -from haystack_integrations.components.embedders.ollama import OllamaDocumentEmbedder from requests import HTTPError +from haystack_integrations.components.embedders.ollama import OllamaDocumentEmbedder + class TestOllamaDocumentEmbedder: def test_init_defaults(self): diff --git a/integrations/ollama/tests/test_generator.py b/integrations/ollama/tests/test_generator.py index 4af2bdb82..069bbd227 100644 --- a/integrations/ollama/tests/test_generator.py +++ b/integrations/ollama/tests/test_generator.py @@ -7,9 +7,10 @@ import pytest from haystack.components.generators.utils import print_streaming_chunk from haystack.dataclasses import StreamingChunk -from haystack_integrations.components.generators.ollama import OllamaGenerator from requests import HTTPError +from haystack_integrations.components.generators.ollama import OllamaGenerator + class TestOllamaGenerator: @pytest.mark.integration diff --git a/integrations/ollama/tests/test_text_embedder.py b/integrations/ollama/tests/test_text_embedder.py index f4d45afec..e7b69460f 100644 --- a/integrations/ollama/tests/test_text_embedder.py +++ b/integrations/ollama/tests/test_text_embedder.py @@ -1,7 +1,8 @@ import pytest -from haystack_integrations.components.embedders.ollama import OllamaTextEmbedder from requests import HTTPError +from haystack_integrations.components.embedders.ollama import OllamaTextEmbedder + class TestOllamaTextEmbedder: def test_init_defaults(self): From ac1cf82bb3fbb25b1c8672933edca7041f99dc58 Mon Sep 17 00:00:00 2001 From: Vladimir Blagojevic Date: Fri, 16 Aug 2024 11:41:04 +0100 Subject: [PATCH 08/25] Update test for the new apply_filter_policy usage (#970) --- integrations/pgvector/tests/test_retrievers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/pgvector/tests/test_retrievers.py b/integrations/pgvector/tests/test_retrievers.py index 031c735fd..058c435ca 100644 --- a/integrations/pgvector/tests/test_retrievers.py +++ b/integrations/pgvector/tests/test_retrievers.py @@ -311,7 +311,7 @@ def test_run_with_filters(self): ) res = retriever.run(query="test query", filters={"field2": "value2"}) - mock_store._keyword_retrieval.assert_called_once_with( + mock_store.mock_calls[0].assert_called_once_with( query="test query", filters={"field": "value", "field2": "value2"}, top_k=10 ) From 4b8fff41e28ed3e258b02e0c6c02544ff418d64b Mon Sep 17 00:00:00 2001 From: Vladimir Blagojevic Date: Fri, 16 Aug 2024 11:42:02 +0100 Subject: [PATCH 09/25] Ruff update, don't ruff tests (#991) Co-authored-by: Madeesh Kannan --- integrations/mongodb_atlas/examples/example.py | 1 + integrations/mongodb_atlas/pyproject.toml | 4 ++-- .../retrievers/mongodb_atlas/embedding_retriever.py | 1 + .../document_stores/mongodb_atlas/document_store.py | 3 ++- integrations/mongodb_atlas/tests/test_document_store.py | 3 ++- integrations/mongodb_atlas/tests/test_embedding_retrieval.py | 1 + integrations/mongodb_atlas/tests/test_retriever.py | 1 + 7 files changed, 10 insertions(+), 4 deletions(-) diff --git a/integrations/mongodb_atlas/examples/example.py b/integrations/mongodb_atlas/examples/example.py index 4cd3edc21..54fd569ce 100644 --- a/integrations/mongodb_atlas/examples/example.py +++ b/integrations/mongodb_atlas/examples/example.py @@ -13,6 +13,7 @@ from haystack.components.embedders import SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder from haystack.components.preprocessors import DocumentSplitter from haystack.components.writers import DocumentWriter + from haystack_integrations.components.retrievers.mongodb_atlas import MongoDBAtlasEmbeddingRetriever from haystack_integrations.document_stores.mongodb_atlas import MongoDBAtlasDocumentStore diff --git a/integrations/mongodb_atlas/pyproject.toml b/integrations/mongodb_atlas/pyproject.toml index 170f6e94d..f8fcf4254 100644 --- a/integrations/mongodb_atlas/pyproject.toml +++ b/integrations/mongodb_atlas/pyproject.toml @@ -66,8 +66,8 @@ detached = true dependencies = ["black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243"] [tool.hatch.envs.lint.scripts] typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}" -style = ["ruff check {args:.}", "black --check --diff {args:.}"] -fmt = ["black {args:.}", "ruff --fix {args:.}", "style"] +style = ["ruff check {args:. --exclude tests/, examples/}", "black --check --diff {args:.}"] +fmt = ["black {args:.}", "ruff --fix {args:. --exclude tests/, examples/}", "style"] all = ["style", "typing"] [tool.black] diff --git a/integrations/mongodb_atlas/src/haystack_integrations/components/retrievers/mongodb_atlas/embedding_retriever.py b/integrations/mongodb_atlas/src/haystack_integrations/components/retrievers/mongodb_atlas/embedding_retriever.py index 91a42e135..3345f4f7c 100644 --- a/integrations/mongodb_atlas/src/haystack_integrations/components/retrievers/mongodb_atlas/embedding_retriever.py +++ b/integrations/mongodb_atlas/src/haystack_integrations/components/retrievers/mongodb_atlas/embedding_retriever.py @@ -7,6 +7,7 @@ from haystack.dataclasses import Document from haystack.document_stores.types import FilterPolicy from haystack.document_stores.types.filter_policy import apply_filter_policy + from haystack_integrations.document_stores.mongodb_atlas import MongoDBAtlasDocumentStore diff --git a/integrations/mongodb_atlas/src/haystack_integrations/document_stores/mongodb_atlas/document_store.py b/integrations/mongodb_atlas/src/haystack_integrations/document_stores/mongodb_atlas/document_store.py index 93eb87005..79caa15f8 100644 --- a/integrations/mongodb_atlas/src/haystack_integrations/document_stores/mongodb_atlas/document_store.py +++ b/integrations/mongodb_atlas/src/haystack_integrations/document_stores/mongodb_atlas/document_store.py @@ -10,12 +10,13 @@ from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError from haystack.document_stores.types import DuplicatePolicy from haystack.utils import Secret, deserialize_secrets_inplace -from haystack_integrations.document_stores.mongodb_atlas.filters import _normalize_filters from pymongo import InsertOne, MongoClient, ReplaceOne, UpdateOne from pymongo.collection import Collection from pymongo.driver_info import DriverInfo from pymongo.errors import BulkWriteError +from haystack_integrations.document_stores.mongodb_atlas.filters import _normalize_filters + logger = logging.getLogger(__name__) diff --git a/integrations/mongodb_atlas/tests/test_document_store.py b/integrations/mongodb_atlas/tests/test_document_store.py index 453d9d16c..6d34b1ca0 100644 --- a/integrations/mongodb_atlas/tests/test_document_store.py +++ b/integrations/mongodb_atlas/tests/test_document_store.py @@ -11,11 +11,12 @@ from haystack.document_stores.types import DuplicatePolicy from haystack.testing.document_store import DocumentStoreBaseTests from haystack.utils import Secret -from haystack_integrations.document_stores.mongodb_atlas import MongoDBAtlasDocumentStore from pandas import DataFrame from pymongo import MongoClient from pymongo.driver_info import DriverInfo +from haystack_integrations.document_stores.mongodb_atlas import MongoDBAtlasDocumentStore + @patch("haystack_integrations.document_stores.mongodb_atlas.document_store.MongoClient") def test_init_is_lazy(_mock_client): diff --git a/integrations/mongodb_atlas/tests/test_embedding_retrieval.py b/integrations/mongodb_atlas/tests/test_embedding_retrieval.py index a03c735e0..143f6e106 100644 --- a/integrations/mongodb_atlas/tests/test_embedding_retrieval.py +++ b/integrations/mongodb_atlas/tests/test_embedding_retrieval.py @@ -6,6 +6,7 @@ import pytest from haystack.document_stores.errors import DocumentStoreError + from haystack_integrations.document_stores.mongodb_atlas import MongoDBAtlasDocumentStore diff --git a/integrations/mongodb_atlas/tests/test_retriever.py b/integrations/mongodb_atlas/tests/test_retriever.py index c25e7487f..832256ccd 100644 --- a/integrations/mongodb_atlas/tests/test_retriever.py +++ b/integrations/mongodb_atlas/tests/test_retriever.py @@ -7,6 +7,7 @@ from haystack.dataclasses import Document from haystack.document_stores.types import FilterPolicy from haystack.utils.auth import EnvVarSecret + from haystack_integrations.components.retrievers.mongodb_atlas import MongoDBAtlasEmbeddingRetriever from haystack_integrations.document_stores.mongodb_atlas import MongoDBAtlasDocumentStore From 37c42639781e28a6fe5f576851e91c2b217c419b Mon Sep 17 00:00:00 2001 From: Vladimir Blagojevic Date: Fri, 16 Aug 2024 11:55:06 +0100 Subject: [PATCH 10/25] Ruff update, don't ruff tests (#984) Co-authored-by: Madeesh Kannan --- integrations/pgvector/examples/embedding_retrieval.py | 1 + integrations/pgvector/examples/hybrid_retrieval.py | 1 + integrations/pgvector/pyproject.toml | 4 ++-- .../components/retrievers/pgvector/embedding_retriever.py | 1 + .../components/retrievers/pgvector/keyword_retriever.py | 1 + integrations/pgvector/tests/conftest.py | 1 + integrations/pgvector/tests/test_document_store.py | 3 ++- integrations/pgvector/tests/test_embedding_retrieval.py | 3 ++- integrations/pgvector/tests/test_filters.py | 7 ++++--- integrations/pgvector/tests/test_keyword_retrieval.py | 1 + integrations/pgvector/tests/test_retrievers.py | 1 + 11 files changed, 17 insertions(+), 7 deletions(-) diff --git a/integrations/pgvector/examples/embedding_retrieval.py b/integrations/pgvector/examples/embedding_retrieval.py index 37ea88929..b20a7a143 100644 --- a/integrations/pgvector/examples/embedding_retrieval.py +++ b/integrations/pgvector/examples/embedding_retrieval.py @@ -17,6 +17,7 @@ from haystack.components.embedders import SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder from haystack.components.preprocessors import DocumentSplitter from haystack.components.writers import DocumentWriter + from haystack_integrations.components.retrievers.pgvector import PgvectorEmbeddingRetriever from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore diff --git a/integrations/pgvector/examples/hybrid_retrieval.py b/integrations/pgvector/examples/hybrid_retrieval.py index cee98fe08..7d110f60e 100644 --- a/integrations/pgvector/examples/hybrid_retrieval.py +++ b/integrations/pgvector/examples/hybrid_retrieval.py @@ -18,6 +18,7 @@ from haystack.components.joiners import DocumentJoiner from haystack.components.preprocessors import DocumentSplitter from haystack.components.writers import DocumentWriter + from haystack_integrations.components.retrievers.pgvector import PgvectorEmbeddingRetriever, PgvectorKeywordRetriever from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore diff --git a/integrations/pgvector/pyproject.toml b/integrations/pgvector/pyproject.toml index 7f31a5203..8f4c2447b 100644 --- a/integrations/pgvector/pyproject.toml +++ b/integrations/pgvector/pyproject.toml @@ -66,8 +66,8 @@ detached = true dependencies = ["black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243"] [tool.hatch.envs.lint.scripts] typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}" -style = ["ruff check {args:.}", "black --check --diff {args:.}"] -fmt = ["black {args:.}", "ruff --fix {args:.}", "style"] +style = ["ruff check {args:. --exclude tests/, examples/}", "black --check --diff {args:.}"] +fmt = ["black {args:.}", "ruff --fix {args:. --exclude tests/, examples/}", "style"] all = ["style", "typing"] [tool.black] diff --git a/integrations/pgvector/src/haystack_integrations/components/retrievers/pgvector/embedding_retriever.py b/integrations/pgvector/src/haystack_integrations/components/retrievers/pgvector/embedding_retriever.py index 22aab1a73..3df5261c6 100644 --- a/integrations/pgvector/src/haystack_integrations/components/retrievers/pgvector/embedding_retriever.py +++ b/integrations/pgvector/src/haystack_integrations/components/retrievers/pgvector/embedding_retriever.py @@ -7,6 +7,7 @@ from haystack.dataclasses import Document from haystack.document_stores.types import FilterPolicy from haystack.document_stores.types.filter_policy import apply_filter_policy + from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore from haystack_integrations.document_stores.pgvector.document_store import VALID_VECTOR_FUNCTIONS diff --git a/integrations/pgvector/src/haystack_integrations/components/retrievers/pgvector/keyword_retriever.py b/integrations/pgvector/src/haystack_integrations/components/retrievers/pgvector/keyword_retriever.py index 636471c31..32f95da64 100644 --- a/integrations/pgvector/src/haystack_integrations/components/retrievers/pgvector/keyword_retriever.py +++ b/integrations/pgvector/src/haystack_integrations/components/retrievers/pgvector/keyword_retriever.py @@ -7,6 +7,7 @@ from haystack.dataclasses import Document from haystack.document_stores.types import FilterPolicy from haystack.document_stores.types.filter_policy import apply_filter_policy + from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore diff --git a/integrations/pgvector/tests/conftest.py b/integrations/pgvector/tests/conftest.py index b53589763..0c3d27088 100644 --- a/integrations/pgvector/tests/conftest.py +++ b/integrations/pgvector/tests/conftest.py @@ -2,6 +2,7 @@ from unittest.mock import patch import pytest + from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore diff --git a/integrations/pgvector/tests/test_document_store.py b/integrations/pgvector/tests/test_document_store.py index eca8190ee..93514b71c 100644 --- a/integrations/pgvector/tests/test_document_store.py +++ b/integrations/pgvector/tests/test_document_store.py @@ -11,9 +11,10 @@ from haystack.document_stores.types import DuplicatePolicy from haystack.testing.document_store import CountDocumentsTest, DeleteDocumentsTest, WriteDocumentsTest from haystack.utils import Secret -from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore from pandas import DataFrame +from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore + @pytest.mark.integration class TestDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDocumentsTest): diff --git a/integrations/pgvector/tests/test_embedding_retrieval.py b/integrations/pgvector/tests/test_embedding_retrieval.py index 2c384f57c..46add79e1 100644 --- a/integrations/pgvector/tests/test_embedding_retrieval.py +++ b/integrations/pgvector/tests/test_embedding_retrieval.py @@ -6,9 +6,10 @@ import pytest from haystack.dataclasses.document import Document -from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore from numpy.random import rand +from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore + @pytest.mark.integration class TestEmbeddingRetrieval: diff --git a/integrations/pgvector/tests/test_filters.py b/integrations/pgvector/tests/test_filters.py index bda10e3c0..08e693471 100644 --- a/integrations/pgvector/tests/test_filters.py +++ b/integrations/pgvector/tests/test_filters.py @@ -3,6 +3,10 @@ import pytest from haystack.dataclasses.document import Document from haystack.testing.document_store import FilterDocumentsTest +from pandas import DataFrame +from psycopg.sql import SQL +from psycopg.types.json import Jsonb + from haystack_integrations.document_stores.pgvector.filters import ( FilterError, _convert_filters_to_where_clause_and_params, @@ -10,9 +14,6 @@ _parse_logical_condition, _treat_meta_field, ) -from pandas import DataFrame -from psycopg.sql import SQL -from psycopg.types.json import Jsonb @pytest.mark.integration diff --git a/integrations/pgvector/tests/test_keyword_retrieval.py b/integrations/pgvector/tests/test_keyword_retrieval.py index 4a5614165..859a61a5f 100644 --- a/integrations/pgvector/tests/test_keyword_retrieval.py +++ b/integrations/pgvector/tests/test_keyword_retrieval.py @@ -1,5 +1,6 @@ import pytest from haystack.dataclasses.document import Document + from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore diff --git a/integrations/pgvector/tests/test_retrievers.py b/integrations/pgvector/tests/test_retrievers.py index 058c435ca..290891307 100644 --- a/integrations/pgvector/tests/test_retrievers.py +++ b/integrations/pgvector/tests/test_retrievers.py @@ -7,6 +7,7 @@ from haystack.dataclasses import Document from haystack.document_stores.types import FilterPolicy from haystack.utils.auth import EnvVarSecret + from haystack_integrations.components.retrievers.pgvector import PgvectorEmbeddingRetriever, PgvectorKeywordRetriever from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore From 0c083df0d6bf056c7d5e401e5fe389fbe5c951b1 Mon Sep 17 00:00:00 2001 From: Madeesh Kannan Date: Fri, 16 Aug 2024 12:56:37 +0200 Subject: [PATCH 11/25] fix: Lints in `nvidia-haystack` (#993) --- integrations/nvidia/pyproject.toml | 17 +++++++++++++---- .../embedders/nvidia/document_embedder.py | 3 ++- .../embedders/nvidia/text_embedder.py | 1 + .../components/generators/nvidia/generator.py | 1 + 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/integrations/nvidia/pyproject.toml b/integrations/nvidia/pyproject.toml index af1d806ef..82fb32b95 100644 --- a/integrations/nvidia/pyproject.toml +++ b/integrations/nvidia/pyproject.toml @@ -42,7 +42,13 @@ root = "../.." git_describe_command = 'git describe --tags --match="integrations/nvidia-v[0-9]*"' [tool.hatch.envs.default] -dependencies = ["coverage[toml]>=6.5", "pytest", "pytest-rerunfailures", "haystack-pydoc-tools", "requests_mock"] +dependencies = [ + "coverage[toml]>=6.5", + "pytest", + "pytest-rerunfailures", + "haystack-pydoc-tools", + "requests_mock", +] [tool.hatch.envs.default.scripts] test = "pytest {args:tests}" test-cov = "coverage run -m pytest {args:tests}" @@ -60,8 +66,11 @@ detached = true dependencies = ["black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243"] [tool.hatch.envs.lint.scripts] typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}" -style = ["ruff check {args:.}", "black --check --diff {args:.}"] -fmt = ["black {args:.}", "ruff --fix {args:.}", "style"] +style = [ + "ruff check {args:. --exclude tests/}", + "black --check --diff {args:.}", +] +fmt = ["black {args:.}", "ruff --fix {args:. --exclude tests/}", "style"] all = ["style", "typing"] [tool.black] @@ -150,7 +159,7 @@ module = [ "pytest.*", "numpy.*", "requests_mock.*", - "pydantic.*" + "pydantic.*", ] ignore_missing_imports = true diff --git a/integrations/nvidia/src/haystack_integrations/components/embedders/nvidia/document_embedder.py b/integrations/nvidia/src/haystack_integrations/components/embedders/nvidia/document_embedder.py index f5d1747b8..52c5393c4 100644 --- a/integrations/nvidia/src/haystack_integrations/components/embedders/nvidia/document_embedder.py +++ b/integrations/nvidia/src/haystack_integrations/components/embedders/nvidia/document_embedder.py @@ -3,9 +3,10 @@ from haystack import Document, component, default_from_dict, default_to_dict from haystack.utils import Secret, deserialize_secrets_inplace -from haystack_integrations.utils.nvidia import NimBackend, is_hosted, url_validation from tqdm import tqdm +from haystack_integrations.utils.nvidia import NimBackend, is_hosted, url_validation + from .truncate import EmbeddingTruncateMode _DEFAULT_API_URL = "https://ai.api.nvidia.com/v1/retrieval/nvidia" diff --git a/integrations/nvidia/src/haystack_integrations/components/embedders/nvidia/text_embedder.py b/integrations/nvidia/src/haystack_integrations/components/embedders/nvidia/text_embedder.py index 1c4a7c5c9..946fc08cb 100644 --- a/integrations/nvidia/src/haystack_integrations/components/embedders/nvidia/text_embedder.py +++ b/integrations/nvidia/src/haystack_integrations/components/embedders/nvidia/text_embedder.py @@ -3,6 +3,7 @@ from haystack import component, default_from_dict, default_to_dict from haystack.utils import Secret, deserialize_secrets_inplace + from haystack_integrations.utils.nvidia import NimBackend, is_hosted, url_validation from .truncate import EmbeddingTruncateMode diff --git a/integrations/nvidia/src/haystack_integrations/components/generators/nvidia/generator.py b/integrations/nvidia/src/haystack_integrations/components/generators/nvidia/generator.py index a286400ab..3eadcc5df 100644 --- a/integrations/nvidia/src/haystack_integrations/components/generators/nvidia/generator.py +++ b/integrations/nvidia/src/haystack_integrations/components/generators/nvidia/generator.py @@ -6,6 +6,7 @@ from haystack import component, default_from_dict, default_to_dict from haystack.utils.auth import Secret, deserialize_secrets_inplace + from haystack_integrations.utils.nvidia import NimBackend, is_hosted, url_validation _DEFAULT_API_URL = "https://integrate.api.nvidia.com/v1" From b3ae89ebad01ab108a9bae527f1c5bf41b192bf8 Mon Sep 17 00:00:00 2001 From: Madeesh Kannan Date: Fri, 16 Aug 2024 12:59:20 +0200 Subject: [PATCH 12/25] fix: Lints in `mistral-haystack` (#994) --- integrations/mistral/examples/indexing_pipeline.py | 1 + .../mistral/examples/streaming_chat_with_rag.py | 1 + integrations/mistral/pyproject.toml | 14 +++++++++++--- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/integrations/mistral/examples/indexing_pipeline.py b/integrations/mistral/examples/indexing_pipeline.py index 0329fab8c..5ce801264 100644 --- a/integrations/mistral/examples/indexing_pipeline.py +++ b/integrations/mistral/examples/indexing_pipeline.py @@ -7,6 +7,7 @@ from haystack.components.preprocessors import DocumentSplitter from haystack.components.writers import DocumentWriter from haystack.document_stores.in_memory import InMemoryDocumentStore + from haystack_integrations.components.embedders.mistral.document_embedder import MistralDocumentEmbedder document_store = InMemoryDocumentStore() diff --git a/integrations/mistral/examples/streaming_chat_with_rag.py b/integrations/mistral/examples/streaming_chat_with_rag.py index 6c7f015d8..776a11796 100644 --- a/integrations/mistral/examples/streaming_chat_with_rag.py +++ b/integrations/mistral/examples/streaming_chat_with_rag.py @@ -11,6 +11,7 @@ from haystack.components.writers import DocumentWriter from haystack.dataclasses import ChatMessage from haystack.document_stores.in_memory import InMemoryDocumentStore + from haystack_integrations.components.embedders.mistral.document_embedder import MistralDocumentEmbedder from haystack_integrations.components.embedders.mistral.text_embedder import MistralTextEmbedder from haystack_integrations.components.generators.mistral import MistralChatGenerator diff --git a/integrations/mistral/pyproject.toml b/integrations/mistral/pyproject.toml index bcb4f5999..8e28c2c06 100644 --- a/integrations/mistral/pyproject.toml +++ b/integrations/mistral/pyproject.toml @@ -41,7 +41,12 @@ root = "../.." git_describe_command = 'git describe --tags --match="integrations/mistral-v[0-9]*"' [tool.hatch.envs.default] -dependencies = ["coverage[toml]>=6.5", "pytest", "pytest-rerunfailures", "haystack-pydoc-tools"] +dependencies = [ + "coverage[toml]>=6.5", + "pytest", + "pytest-rerunfailures", + "haystack-pydoc-tools", +] [tool.hatch.envs.default.scripts] test = "pytest {args:tests}" test-cov = "coverage run -m pytest {args:tests}" @@ -59,8 +64,11 @@ detached = true dependencies = ["black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243"] [tool.hatch.envs.lint.scripts] typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}" -style = ["ruff check {args:.}", "black --check --diff {args:.}"] -fmt = ["black {args:.}", "ruff --fix {args:.}", "style"] +style = [ + "ruff check {args:. --exclude tests/}", + "black --check --diff {args:.}", +] +fmt = ["black {args:.}", "ruff --fix {args:. --exclude tests/}", "style"] all = ["style", "typing"] [tool.black] From 91d2acf8a69fd0d22ce06efc0c12f2af6a577cfc Mon Sep 17 00:00:00 2001 From: Vladimir Blagojevic Date: Fri, 16 Aug 2024 12:09:22 +0100 Subject: [PATCH 13/25] Ruff update, don't ruff tests (#997) --- integrations/fastembed/examples/example.py | 1 + integrations/fastembed/examples/sparse_example.py | 1 + integrations/fastembed/pyproject.toml | 4 ++-- .../fastembed/tests/test_fastembed_document_embedder.py | 1 + .../tests/test_fastembed_sparse_document_embedder.py | 1 + .../fastembed/tests/test_fastembed_sparse_text_embedder.py | 1 + integrations/fastembed/tests/test_fastembed_text_embedder.py | 1 + 7 files changed, 8 insertions(+), 2 deletions(-) diff --git a/integrations/fastembed/examples/example.py b/integrations/fastembed/examples/example.py index e4d328210..640abd33b 100644 --- a/integrations/fastembed/examples/example.py +++ b/integrations/fastembed/examples/example.py @@ -1,6 +1,7 @@ from haystack import Document, Pipeline from haystack.components.retrievers.in_memory import InMemoryEmbeddingRetriever from haystack.document_stores.in_memory import InMemoryDocumentStore + from haystack_integrations.components.embedders.fastembed import FastembedDocumentEmbedder, FastembedTextEmbedder document_store = InMemoryDocumentStore(embedding_similarity_function="cosine") diff --git a/integrations/fastembed/examples/sparse_example.py b/integrations/fastembed/examples/sparse_example.py index bce3b363d..206788f18 100644 --- a/integrations/fastembed/examples/sparse_example.py +++ b/integrations/fastembed/examples/sparse_example.py @@ -4,6 +4,7 @@ # involving indexing and retrieval of documents. from haystack import Document + from haystack_integrations.components.embedders.fastembed import FastembedSparseDocumentEmbedder document_list = [ diff --git a/integrations/fastembed/pyproject.toml b/integrations/fastembed/pyproject.toml index 9afd344c9..4ebd765dd 100644 --- a/integrations/fastembed/pyproject.toml +++ b/integrations/fastembed/pyproject.toml @@ -66,8 +66,8 @@ detached = true dependencies = ["black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243", "numpy"] [tool.hatch.envs.lint.scripts] typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}" -style = ["ruff check {args:.}", "black --check --diff {args:.}"] -fmt = ["black {args:.}", "ruff --fix {args:.}", "style"] +style = ["ruff check {args:. --exclude tests/, examples/}", "black --check --diff {args:.}"] +fmt = ["black {args:.}", "ruff --fix {args:. --exclude tests/, examples/}", "style"] all = ["style", "typing"] [tool.black] diff --git a/integrations/fastembed/tests/test_fastembed_document_embedder.py b/integrations/fastembed/tests/test_fastembed_document_embedder.py index 8afb89c69..8a7589de1 100644 --- a/integrations/fastembed/tests/test_fastembed_document_embedder.py +++ b/integrations/fastembed/tests/test_fastembed_document_embedder.py @@ -3,6 +3,7 @@ import numpy as np import pytest from haystack import Document, default_from_dict + from haystack_integrations.components.embedders.fastembed.fastembed_document_embedder import ( FastembedDocumentEmbedder, ) diff --git a/integrations/fastembed/tests/test_fastembed_sparse_document_embedder.py b/integrations/fastembed/tests/test_fastembed_sparse_document_embedder.py index b4caca364..d3f2023b8 100644 --- a/integrations/fastembed/tests/test_fastembed_sparse_document_embedder.py +++ b/integrations/fastembed/tests/test_fastembed_sparse_document_embedder.py @@ -4,6 +4,7 @@ import pytest from haystack import Document, default_from_dict from haystack.dataclasses.sparse_embedding import SparseEmbedding + from haystack_integrations.components.embedders.fastembed.fastembed_sparse_document_embedder import ( FastembedSparseDocumentEmbedder, ) diff --git a/integrations/fastembed/tests/test_fastembed_sparse_text_embedder.py b/integrations/fastembed/tests/test_fastembed_sparse_text_embedder.py index 9e37df409..7e9197493 100644 --- a/integrations/fastembed/tests/test_fastembed_sparse_text_embedder.py +++ b/integrations/fastembed/tests/test_fastembed_sparse_text_embedder.py @@ -4,6 +4,7 @@ import pytest from haystack import default_from_dict from haystack.dataclasses.sparse_embedding import SparseEmbedding + from haystack_integrations.components.embedders.fastembed.fastembed_sparse_text_embedder import ( FastembedSparseTextEmbedder, ) diff --git a/integrations/fastembed/tests/test_fastembed_text_embedder.py b/integrations/fastembed/tests/test_fastembed_text_embedder.py index f20a98b57..f1b2e21e9 100644 --- a/integrations/fastembed/tests/test_fastembed_text_embedder.py +++ b/integrations/fastembed/tests/test_fastembed_text_embedder.py @@ -3,6 +3,7 @@ import numpy as np import pytest from haystack import default_from_dict + from haystack_integrations.components.embedders.fastembed.fastembed_text_embedder import ( FastembedTextEmbedder, ) From 7885aa138d6452e31a99abd8f2b5e37b924f31a5 Mon Sep 17 00:00:00 2001 From: Madeesh Kannan Date: Fri, 16 Aug 2024 13:09:35 +0200 Subject: [PATCH 14/25] fix: Lints in `cohere-haystack` (#995) --- integrations/cohere/examples/cohere_embedding.py | 1 + integrations/cohere/examples/cohere_generation.py | 1 + integrations/cohere/examples/cohere_ranker.py | 1 + integrations/cohere/pyproject.toml | 14 +++++++++++--- .../embedders/cohere/document_embedder.py | 2 +- .../components/embedders/cohere/text_embedder.py | 2 +- 6 files changed, 16 insertions(+), 5 deletions(-) diff --git a/integrations/cohere/examples/cohere_embedding.py b/integrations/cohere/examples/cohere_embedding.py index e6fe3cc35..75131de86 100644 --- a/integrations/cohere/examples/cohere_embedding.py +++ b/integrations/cohere/examples/cohere_embedding.py @@ -1,6 +1,7 @@ from haystack import Document, Pipeline from haystack.components.retrievers.in_memory import InMemoryEmbeddingRetriever from haystack.document_stores.in_memory import InMemoryDocumentStore + from haystack_integrations.components.embedders.cohere.document_embedder import CohereDocumentEmbedder from haystack_integrations.components.embedders.cohere.text_embedder import CohereTextEmbedder diff --git a/integrations/cohere/examples/cohere_generation.py b/integrations/cohere/examples/cohere_generation.py index cd79e37d3..3baec6327 100644 --- a/integrations/cohere/examples/cohere_generation.py +++ b/integrations/cohere/examples/cohere_generation.py @@ -20,6 +20,7 @@ from haystack.components.joiners import BranchJoiner from haystack.components.validators import JsonSchemaValidator from haystack.dataclasses import ChatMessage + from haystack_integrations.components.generators.cohere import CohereChatGenerator # Defines a JSON schema for validating a person's data. The schema specifies that a valid object must diff --git a/integrations/cohere/examples/cohere_ranker.py b/integrations/cohere/examples/cohere_ranker.py index 79a3d346d..b66ca1aae 100644 --- a/integrations/cohere/examples/cohere_ranker.py +++ b/integrations/cohere/examples/cohere_ranker.py @@ -1,6 +1,7 @@ from haystack import Document, Pipeline from haystack.components.retrievers.in_memory import InMemoryBM25Retriever from haystack.document_stores.in_memory import InMemoryDocumentStore + from haystack_integrations.components.rankers.cohere import CohereRanker # Note set your API key by running the below command in your terminal diff --git a/integrations/cohere/pyproject.toml b/integrations/cohere/pyproject.toml index 04fe15585..add8dc150 100644 --- a/integrations/cohere/pyproject.toml +++ b/integrations/cohere/pyproject.toml @@ -41,7 +41,12 @@ root = "../.." git_describe_command = 'git describe --tags --match="integrations/cohere-v[0-9]*"' [tool.hatch.envs.default] -dependencies = ["coverage[toml]>=6.5", "pytest", "pytest-rerunfailures", "haystack-pydoc-tools"] +dependencies = [ + "coverage[toml]>=6.5", + "pytest", + "pytest-rerunfailures", + "haystack-pydoc-tools", +] [tool.hatch.envs.default.scripts] test = "pytest {args:tests}" test-cov = "coverage run -m pytest {args:tests}" @@ -59,8 +64,11 @@ detached = true dependencies = ["black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243"] [tool.hatch.envs.lint.scripts] typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}" -style = ["ruff check {args:.}", "black --check --diff {args:.}"] -fmt = ["black {args:.}", "ruff --fix {args:.}", "style"] +style = [ + "ruff check {args:. --exclude tests/}", + "black --check --diff {args:.}", +] +fmt = ["black {args:.}", "ruff --fix {args:. --exclude tests/}", "style"] all = ["style", "typing"] [tool.black] diff --git a/integrations/cohere/src/haystack_integrations/components/embedders/cohere/document_embedder.py b/integrations/cohere/src/haystack_integrations/components/embedders/cohere/document_embedder.py index 59a04cf3c..3201168a8 100644 --- a/integrations/cohere/src/haystack_integrations/components/embedders/cohere/document_embedder.py +++ b/integrations/cohere/src/haystack_integrations/components/embedders/cohere/document_embedder.py @@ -6,9 +6,9 @@ from haystack import Document, component, default_from_dict, default_to_dict from haystack.utils import Secret, deserialize_secrets_inplace -from haystack_integrations.components.embedders.cohere.utils import get_async_response, get_response from cohere import AsyncClient, Client +from haystack_integrations.components.embedders.cohere.utils import get_async_response, get_response @component diff --git a/integrations/cohere/src/haystack_integrations/components/embedders/cohere/text_embedder.py b/integrations/cohere/src/haystack_integrations/components/embedders/cohere/text_embedder.py index 80ede51bf..c1e9bd613 100644 --- a/integrations/cohere/src/haystack_integrations/components/embedders/cohere/text_embedder.py +++ b/integrations/cohere/src/haystack_integrations/components/embedders/cohere/text_embedder.py @@ -6,9 +6,9 @@ from haystack import component, default_from_dict, default_to_dict from haystack.utils import Secret, deserialize_secrets_inplace -from haystack_integrations.components.embedders.cohere.utils import get_async_response, get_response from cohere import AsyncClient, Client +from haystack_integrations.components.embedders.cohere.utils import get_async_response, get_response @component From 43389f47bb75bb0386381c2a50970a2b0c7b1f4d Mon Sep 17 00:00:00 2001 From: Vladimir Blagojevic Date: Fri, 16 Aug 2024 12:11:57 +0100 Subject: [PATCH 15/25] Ruff update, don't ruff tests (#999) --- integrations/elasticsearch/pyproject.toml | 4 ++-- .../components/retrievers/elasticsearch/bm25_retriever.py | 1 + .../retrievers/elasticsearch/embedding_retriever.py | 1 + integrations/elasticsearch/tests/test_bm25_retriever.py | 1 + integrations/elasticsearch/tests/test_document_store.py | 1 + integrations/elasticsearch/tests/test_embedding_retriever.py | 1 + integrations/elasticsearch/tests/test_filters.py | 1 + 7 files changed, 8 insertions(+), 2 deletions(-) diff --git a/integrations/elasticsearch/pyproject.toml b/integrations/elasticsearch/pyproject.toml index cb9281030..4e13b1c23 100644 --- a/integrations/elasticsearch/pyproject.toml +++ b/integrations/elasticsearch/pyproject.toml @@ -65,8 +65,8 @@ detached = true dependencies = ["black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243"] [tool.hatch.envs.lint.scripts] typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}" -style = ["ruff check {args:.}", "black --check --diff {args:.}"] -fmt = ["black {args:.}", "ruff --fix {args:.}", "style"] +style = ["ruff check {args:. --exclude tests/}", "black --check --diff {args:.}"] +fmt = ["black {args:.}", "ruff --fix {args:. --exclude tests/}", "style"] all = ["style", "typing"] [tool.hatch.metadata] diff --git a/integrations/elasticsearch/src/haystack_integrations/components/retrievers/elasticsearch/bm25_retriever.py b/integrations/elasticsearch/src/haystack_integrations/components/retrievers/elasticsearch/bm25_retriever.py index f273c955b..c25bd1e81 100644 --- a/integrations/elasticsearch/src/haystack_integrations/components/retrievers/elasticsearch/bm25_retriever.py +++ b/integrations/elasticsearch/src/haystack_integrations/components/retrievers/elasticsearch/bm25_retriever.py @@ -7,6 +7,7 @@ from haystack.dataclasses import Document from haystack.document_stores.types import FilterPolicy from haystack.document_stores.types.filter_policy import apply_filter_policy + from haystack_integrations.document_stores.elasticsearch.document_store import ElasticsearchDocumentStore diff --git a/integrations/elasticsearch/src/haystack_integrations/components/retrievers/elasticsearch/embedding_retriever.py b/integrations/elasticsearch/src/haystack_integrations/components/retrievers/elasticsearch/embedding_retriever.py index 10e860ea4..b8cebc544 100644 --- a/integrations/elasticsearch/src/haystack_integrations/components/retrievers/elasticsearch/embedding_retriever.py +++ b/integrations/elasticsearch/src/haystack_integrations/components/retrievers/elasticsearch/embedding_retriever.py @@ -7,6 +7,7 @@ from haystack.dataclasses import Document from haystack.document_stores.types import FilterPolicy from haystack.document_stores.types.filter_policy import apply_filter_policy + from haystack_integrations.document_stores.elasticsearch.document_store import ElasticsearchDocumentStore diff --git a/integrations/elasticsearch/tests/test_bm25_retriever.py b/integrations/elasticsearch/tests/test_bm25_retriever.py index 3e9ebc9b8..c306c00b7 100644 --- a/integrations/elasticsearch/tests/test_bm25_retriever.py +++ b/integrations/elasticsearch/tests/test_bm25_retriever.py @@ -6,6 +6,7 @@ import pytest from haystack.dataclasses import Document from haystack.document_stores.types import FilterPolicy + from haystack_integrations.components.retrievers.elasticsearch import ElasticsearchBM25Retriever from haystack_integrations.document_stores.elasticsearch import ElasticsearchDocumentStore diff --git a/integrations/elasticsearch/tests/test_document_store.py b/integrations/elasticsearch/tests/test_document_store.py index 20b68f126..51a19b641 100644 --- a/integrations/elasticsearch/tests/test_document_store.py +++ b/integrations/elasticsearch/tests/test_document_store.py @@ -12,6 +12,7 @@ from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError from haystack.document_stores.types import DuplicatePolicy from haystack.testing.document_store import DocumentStoreBaseTests + from haystack_integrations.document_stores.elasticsearch import ElasticsearchDocumentStore diff --git a/integrations/elasticsearch/tests/test_embedding_retriever.py b/integrations/elasticsearch/tests/test_embedding_retriever.py index 2d03f0ec2..0725548bf 100644 --- a/integrations/elasticsearch/tests/test_embedding_retriever.py +++ b/integrations/elasticsearch/tests/test_embedding_retriever.py @@ -6,6 +6,7 @@ import pytest from haystack.dataclasses import Document from haystack.document_stores.types import FilterPolicy + from haystack_integrations.components.retrievers.elasticsearch import ElasticsearchEmbeddingRetriever from haystack_integrations.document_stores.elasticsearch import ElasticsearchDocumentStore diff --git a/integrations/elasticsearch/tests/test_filters.py b/integrations/elasticsearch/tests/test_filters.py index 86e5cba74..5f09f0a49 100644 --- a/integrations/elasticsearch/tests/test_filters.py +++ b/integrations/elasticsearch/tests/test_filters.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 import pytest from haystack.errors import FilterError + from haystack_integrations.document_stores.elasticsearch.filters import _normalize_filters, _normalize_ranges filters_data = [ From b293b76c166c3537ebec96b3b17bfad58af2546d Mon Sep 17 00:00:00 2001 From: Marc Klingen Date: Fri, 16 Aug 2024 13:16:16 +0200 Subject: [PATCH 16/25] docs: add link to langfuse in LangfuseConnector (#981) Co-authored-by: Vladimir Blagojevic --- .../components/connectors/langfuse/langfuse_connector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integrations/langfuse/src/haystack_integrations/components/connectors/langfuse/langfuse_connector.py b/integrations/langfuse/src/haystack_integrations/components/connectors/langfuse/langfuse_connector.py index ab93c3433..29f58d722 100644 --- a/integrations/langfuse/src/haystack_integrations/components/connectors/langfuse/langfuse_connector.py +++ b/integrations/langfuse/src/haystack_integrations/components/connectors/langfuse/langfuse_connector.py @@ -7,8 +7,8 @@ @component class LangfuseConnector: """ - LangfuseConnector connects Haystack LLM framework with Langfuse in order to enable the tracing of operations - and data flow within various components of a pipeline. + LangfuseConnector connects Haystack LLM framework with [Langfuse](https://langfuse.com) in order to enable the + tracing of operations and data flow within various components of a pipeline. Simply add this component to your pipeline, but *do not* connect it to any other component. The LangfuseConnector will automatically trace the operations and data flow within the pipeline. From 03ce349ec2480c727581095c13c136a6356834e0 Mon Sep 17 00:00:00 2001 From: Madeesh Kannan Date: Fri, 16 Aug 2024 13:35:44 +0200 Subject: [PATCH 17/25] fix: Lints in `unstructured-haystack` (#996) --- integrations/unstructured/pyproject.toml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/integrations/unstructured/pyproject.toml b/integrations/unstructured/pyproject.toml index b5de6c66a..6811753d9 100644 --- a/integrations/unstructured/pyproject.toml +++ b/integrations/unstructured/pyproject.toml @@ -64,8 +64,11 @@ detached = true dependencies = ["black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243"] [tool.hatch.envs.lint.scripts] typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}" -style = ["ruff check {args:.}", "black --check --diff {args:.}"] -fmt = ["black {args:.}", "ruff --fix {args:.}", "style"] +style = [ + "ruff check {args:. --exclude tests/}", + "black --check --diff {args:.}", +] +fmt = ["black {args:.}", "ruff --fix {args:. --exclude tests/}", "style"] all = ["style", "typing"] [tool.hatch.metadata] From 58c01836806cf4b9f3e05b40e95a4516fa6eae44 Mon Sep 17 00:00:00 2001 From: Vladimir Blagojevic Date: Fri, 16 Aug 2024 12:38:14 +0100 Subject: [PATCH 18/25] Ruff update, don't ruff tests (#998) --- integrations/llama_cpp/examples/rag_pipeline_example.py | 1 + integrations/llama_cpp/pyproject.toml | 4 ++-- integrations/llama_cpp/tests/test_chat_generator.py | 1 + integrations/llama_cpp/tests/test_generator.py | 1 + 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/integrations/llama_cpp/examples/rag_pipeline_example.py b/integrations/llama_cpp/examples/rag_pipeline_example.py index dcab6dbd6..01ee11b98 100644 --- a/integrations/llama_cpp/examples/rag_pipeline_example.py +++ b/integrations/llama_cpp/examples/rag_pipeline_example.py @@ -6,6 +6,7 @@ from haystack.components.retrievers import InMemoryEmbeddingRetriever from haystack.components.writers import DocumentWriter from haystack.document_stores import InMemoryDocumentStore + from haystack_integrations.components.generators.llama_cpp import LlamaCppGenerator # Load first 100 rows of the Simple Wikipedia Dataset from HuggingFace diff --git a/integrations/llama_cpp/pyproject.toml b/integrations/llama_cpp/pyproject.toml index 2efb15d53..d57bd9fc9 100644 --- a/integrations/llama_cpp/pyproject.toml +++ b/integrations/llama_cpp/pyproject.toml @@ -70,8 +70,8 @@ dependencies = ["black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243"] [tool.hatch.envs.lint.scripts] typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}" -style = ["ruff check {args:.}", "black --check --diff {args:.}"] -fmt = ["black {args:.}", "ruff --fix {args:.}", "style"] +style = ["ruff check {args:. --exclude tests/, examples/}", "black --check --diff {args:.}"] +fmt = ["black {args:.}", "ruff --fix {args:. --exclude tests/, examples/}", "style"] all = ["style", "typing"] [tool.hatch.metadata] diff --git a/integrations/llama_cpp/tests/test_chat_generator.py b/integrations/llama_cpp/tests/test_chat_generator.py index 7bd6ef122..1d4c9cf82 100644 --- a/integrations/llama_cpp/tests/test_chat_generator.py +++ b/integrations/llama_cpp/tests/test_chat_generator.py @@ -10,6 +10,7 @@ from haystack.components.retrievers.in_memory import InMemoryBM25Retriever from haystack.dataclasses import ChatMessage, ChatRole from haystack.document_stores.in_memory import InMemoryDocumentStore + from haystack_integrations.components.generators.llama_cpp.chat.chat_generator import ( LlamaCppChatGenerator, _convert_message_to_llamacpp_format, diff --git a/integrations/llama_cpp/tests/test_generator.py b/integrations/llama_cpp/tests/test_generator.py index 04b8339e5..f57de94b1 100644 --- a/integrations/llama_cpp/tests/test_generator.py +++ b/integrations/llama_cpp/tests/test_generator.py @@ -9,6 +9,7 @@ from haystack.components.builders.prompt_builder import PromptBuilder from haystack.components.retrievers.in_memory import InMemoryBM25Retriever from haystack.document_stores.in_memory import InMemoryDocumentStore + from haystack_integrations.components.generators.llama_cpp import LlamaCppGenerator From db5112eb0a3daab34d8f528e772fb4f5b31f6a99 Mon Sep 17 00:00:00 2001 From: Vladimir Blagojevic Date: Fri, 16 Aug 2024 12:38:27 +0100 Subject: [PATCH 19/25] Ruff update, don't ruff tests (#989) --- integrations/qdrant/examples/embedding_retrieval.py | 1 + integrations/qdrant/pyproject.toml | 4 ++-- .../components/retrievers/qdrant/retriever.py | 3 ++- .../document_stores/qdrant/migrate_to_sparse.py | 3 ++- integrations/qdrant/tests/test_converters.py | 3 ++- integrations/qdrant/tests/test_dict_converters.py | 1 + integrations/qdrant/tests/test_document_store.py | 3 ++- integrations/qdrant/tests/test_filters.py | 3 ++- integrations/qdrant/tests/test_legacy_filters.py | 1 + integrations/qdrant/tests/test_retriever.py | 1 + 10 files changed, 16 insertions(+), 7 deletions(-) diff --git a/integrations/qdrant/examples/embedding_retrieval.py b/integrations/qdrant/examples/embedding_retrieval.py index f009191e7..84f12f4fe 100644 --- a/integrations/qdrant/examples/embedding_retrieval.py +++ b/integrations/qdrant/examples/embedding_retrieval.py @@ -12,6 +12,7 @@ from haystack.components.embedders import SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder from haystack.components.preprocessors import DocumentSplitter from haystack.components.writers import DocumentWriter + from haystack_integrations.components.retrievers.qdrant import QdrantEmbeddingRetriever from haystack_integrations.document_stores.qdrant import QdrantDocumentStore diff --git a/integrations/qdrant/pyproject.toml b/integrations/qdrant/pyproject.toml index 8b9c44cc7..225844f22 100644 --- a/integrations/qdrant/pyproject.toml +++ b/integrations/qdrant/pyproject.toml @@ -62,8 +62,8 @@ detached = true dependencies = ["black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243"] [tool.hatch.envs.lint.scripts] typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}" -style = ["ruff check {args:.}", "black --check --diff {args:.}"] -fmt = ["black {args:.}", "ruff --fix {args:.}", "style"] +style = ["ruff check {args:. --exclude tests/, examples/}", "black --check --diff {args:.}"] +fmt = ["black {args:.}", "ruff --fix {args:. --exclude tests/, examples/}", "style"] all = ["style", "typing"] [tool.black] diff --git a/integrations/qdrant/src/haystack_integrations/components/retrievers/qdrant/retriever.py b/integrations/qdrant/src/haystack_integrations/components/retrievers/qdrant/retriever.py index 275a46f95..408b2458a 100644 --- a/integrations/qdrant/src/haystack_integrations/components/retrievers/qdrant/retriever.py +++ b/integrations/qdrant/src/haystack_integrations/components/retrievers/qdrant/retriever.py @@ -4,9 +4,10 @@ from haystack.dataclasses.sparse_embedding import SparseEmbedding from haystack.document_stores.types import FilterPolicy from haystack.document_stores.types.filter_policy import apply_filter_policy -from haystack_integrations.document_stores.qdrant import QdrantDocumentStore from qdrant_client.http import models +from haystack_integrations.document_stores.qdrant import QdrantDocumentStore + @component class QdrantEmbeddingRetriever: diff --git a/integrations/qdrant/src/haystack_integrations/document_stores/qdrant/migrate_to_sparse.py b/integrations/qdrant/src/haystack_integrations/document_stores/qdrant/migrate_to_sparse.py index 1fabbfd9c..9eb4ca088 100644 --- a/integrations/qdrant/src/haystack_integrations/document_stores/qdrant/migrate_to_sparse.py +++ b/integrations/qdrant/src/haystack_integrations/document_stores/qdrant/migrate_to_sparse.py @@ -1,9 +1,10 @@ import logging import time -from haystack_integrations.document_stores.qdrant import QdrantDocumentStore from qdrant_client.http import models +from haystack_integrations.document_stores.qdrant import QdrantDocumentStore + logger = logging.getLogger(__name__) logger.addHandler(logging.StreamHandler()) logger.setLevel(logging.INFO) diff --git a/integrations/qdrant/tests/test_converters.py b/integrations/qdrant/tests/test_converters.py index 242c4cafe..566da49ab 100644 --- a/integrations/qdrant/tests/test_converters.py +++ b/integrations/qdrant/tests/test_converters.py @@ -1,9 +1,10 @@ import numpy as np +from qdrant_client.http import models as rest + from haystack_integrations.document_stores.qdrant.converters import ( convert_id, convert_qdrant_point_to_haystack_document, ) -from qdrant_client.http import models as rest def test_convert_id_is_deterministic(): diff --git a/integrations/qdrant/tests/test_dict_converters.py b/integrations/qdrant/tests/test_dict_converters.py index 3871dbff0..8ea5b5595 100644 --- a/integrations/qdrant/tests/test_dict_converters.py +++ b/integrations/qdrant/tests/test_dict_converters.py @@ -1,4 +1,5 @@ from haystack.utils import Secret + from haystack_integrations.document_stores.qdrant import QdrantDocumentStore diff --git a/integrations/qdrant/tests/test_document_store.py b/integrations/qdrant/tests/test_document_store.py index c388a10cf..5559d7bac 100644 --- a/integrations/qdrant/tests/test_document_store.py +++ b/integrations/qdrant/tests/test_document_store.py @@ -12,12 +12,13 @@ WriteDocumentsTest, _random_embeddings, ) +from qdrant_client.http import models as rest + from haystack_integrations.document_stores.qdrant.document_store import ( SPARSE_VECTORS_NAME, QdrantDocumentStore, QdrantStoreError, ) -from qdrant_client.http import models as rest class TestQdrantDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDocumentsTest): diff --git a/integrations/qdrant/tests/test_filters.py b/integrations/qdrant/tests/test_filters.py index fd070bda9..1afd2b7f3 100644 --- a/integrations/qdrant/tests/test_filters.py +++ b/integrations/qdrant/tests/test_filters.py @@ -4,9 +4,10 @@ from haystack import Document from haystack.testing.document_store import FilterDocumentsTest from haystack.utils.filters import FilterError -from haystack_integrations.document_stores.qdrant import QdrantDocumentStore from qdrant_client.http import models +from haystack_integrations.document_stores.qdrant import QdrantDocumentStore + class TestQdrantStoreBaseTests(FilterDocumentsTest): @pytest.fixture diff --git a/integrations/qdrant/tests/test_legacy_filters.py b/integrations/qdrant/tests/test_legacy_filters.py index 60f1fad2b..a09f042c5 100644 --- a/integrations/qdrant/tests/test_legacy_filters.py +++ b/integrations/qdrant/tests/test_legacy_filters.py @@ -5,6 +5,7 @@ from haystack.document_stores.types import DocumentStore from haystack.testing.document_store import LegacyFilterDocumentsTest from haystack.utils.filters import FilterError + from haystack_integrations.document_stores.qdrant import QdrantDocumentStore # The tests below are from haystack.testing.document_store.LegacyFilterDocumentsTest diff --git a/integrations/qdrant/tests/test_retriever.py b/integrations/qdrant/tests/test_retriever.py index 5a558db3b..eb0386828 100644 --- a/integrations/qdrant/tests/test_retriever.py +++ b/integrations/qdrant/tests/test_retriever.py @@ -8,6 +8,7 @@ FilterableDocsFixtureMixin, _random_embeddings, ) + from haystack_integrations.components.retrievers.qdrant import ( QdrantEmbeddingRetriever, QdrantHybridRetriever, From f08d57f1d2bbbcd7ce8b9e3799eb7d9560238790 Mon Sep 17 00:00:00 2001 From: Vladimir Blagojevic Date: Fri, 16 Aug 2024 12:38:44 +0100 Subject: [PATCH 20/25] Ruff update, don't ruff tests (#988) --- integrations/opensearch/pyproject.toml | 4 ++-- .../components/retrievers/opensearch/bm25_retriever.py | 1 + .../components/retrievers/opensearch/embedding_retriever.py | 1 + .../document_stores/opensearch/document_store.py | 5 +++-- integrations/opensearch/tests/test_auth.py | 3 ++- integrations/opensearch/tests/test_bm25_retriever.py | 1 + integrations/opensearch/tests/test_document_store.py | 3 ++- integrations/opensearch/tests/test_embedding_retriever.py | 1 + integrations/opensearch/tests/test_filters.py | 1 + 9 files changed, 14 insertions(+), 6 deletions(-) diff --git a/integrations/opensearch/pyproject.toml b/integrations/opensearch/pyproject.toml index b7e5e3da6..6be86727e 100644 --- a/integrations/opensearch/pyproject.toml +++ b/integrations/opensearch/pyproject.toml @@ -67,8 +67,8 @@ detached = true dependencies = ["black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243", "boto3"] [tool.hatch.envs.lint.scripts] typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}" -style = ["ruff check {args:.}", "black --check --diff {args:.}"] -fmt = ["black {args:.}", "ruff check --fix {args:.}", "style"] +style = ["ruff check {args:. --exclude tests/}", "black --check --diff {args:.}"] +fmt = ["black {args:.}", "ruff check --fix {args:. --exclude tests/}", "style"] all = ["style", "typing"] [tool.hatch.metadata] diff --git a/integrations/opensearch/src/haystack_integrations/components/retrievers/opensearch/bm25_retriever.py b/integrations/opensearch/src/haystack_integrations/components/retrievers/opensearch/bm25_retriever.py index 640f349b2..f5d45d2b8 100644 --- a/integrations/opensearch/src/haystack_integrations/components/retrievers/opensearch/bm25_retriever.py +++ b/integrations/opensearch/src/haystack_integrations/components/retrievers/opensearch/bm25_retriever.py @@ -8,6 +8,7 @@ from haystack.dataclasses import Document from haystack.document_stores.types import FilterPolicy from haystack.document_stores.types.filter_policy import apply_filter_policy + from haystack_integrations.document_stores.opensearch import OpenSearchDocumentStore logger = logging.getLogger(__name__) diff --git a/integrations/opensearch/src/haystack_integrations/components/retrievers/opensearch/embedding_retriever.py b/integrations/opensearch/src/haystack_integrations/components/retrievers/opensearch/embedding_retriever.py index cdf905b97..1c1071a76 100644 --- a/integrations/opensearch/src/haystack_integrations/components/retrievers/opensearch/embedding_retriever.py +++ b/integrations/opensearch/src/haystack_integrations/components/retrievers/opensearch/embedding_retriever.py @@ -8,6 +8,7 @@ from haystack.dataclasses import Document from haystack.document_stores.types import FilterPolicy from haystack.document_stores.types.filter_policy import apply_filter_policy + from haystack_integrations.document_stores.opensearch import OpenSearchDocumentStore logger = logging.getLogger(__name__) diff --git a/integrations/opensearch/src/haystack_integrations/document_stores/opensearch/document_store.py b/integrations/opensearch/src/haystack_integrations/document_stores/opensearch/document_store.py index 465897608..3a6056bd2 100644 --- a/integrations/opensearch/src/haystack_integrations/document_stores/opensearch/document_store.py +++ b/integrations/opensearch/src/haystack_integrations/document_stores/opensearch/document_store.py @@ -10,11 +10,12 @@ from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError from haystack.document_stores.types import DuplicatePolicy from haystack.utils.filters import convert -from haystack_integrations.document_stores.opensearch.auth import AWSAuth -from haystack_integrations.document_stores.opensearch.filters import normalize_filters from opensearchpy import OpenSearch from opensearchpy.helpers import bulk +from haystack_integrations.document_stores.opensearch.auth import AWSAuth +from haystack_integrations.document_stores.opensearch.filters import normalize_filters + logger = logging.getLogger(__name__) Hosts = Union[str, List[Union[str, Mapping[str, Union[str, int]]]]] diff --git a/integrations/opensearch/tests/test_auth.py b/integrations/opensearch/tests/test_auth.py index 25bda7d66..a413b9b56 100644 --- a/integrations/opensearch/tests/test_auth.py +++ b/integrations/opensearch/tests/test_auth.py @@ -1,9 +1,10 @@ from unittest.mock import Mock, patch import pytest -from haystack_integrations.document_stores.opensearch.auth import AWSAuth from opensearchpy import Urllib3AWSV4SignerAuth +from haystack_integrations.document_stores.opensearch.auth import AWSAuth + class TestAWSAuth: @pytest.fixture(autouse=True) diff --git a/integrations/opensearch/tests/test_bm25_retriever.py b/integrations/opensearch/tests/test_bm25_retriever.py index c015d360a..ef3275608 100644 --- a/integrations/opensearch/tests/test_bm25_retriever.py +++ b/integrations/opensearch/tests/test_bm25_retriever.py @@ -6,6 +6,7 @@ import pytest from haystack.dataclasses import Document from haystack.document_stores.types import FilterPolicy + from haystack_integrations.components.retrievers.opensearch import OpenSearchBM25Retriever from haystack_integrations.document_stores.opensearch import OpenSearchDocumentStore from haystack_integrations.document_stores.opensearch.document_store import DEFAULT_MAX_CHUNK_BYTES diff --git a/integrations/opensearch/tests/test_document_store.py b/integrations/opensearch/tests/test_document_store.py index 287c24f63..a7d516b3e 100644 --- a/integrations/opensearch/tests/test_document_store.py +++ b/integrations/opensearch/tests/test_document_store.py @@ -11,10 +11,11 @@ from haystack.document_stores.types import DuplicatePolicy from haystack.testing.document_store import DocumentStoreBaseTests from haystack.utils.auth import Secret +from opensearchpy.exceptions import RequestError + from haystack_integrations.document_stores.opensearch import OpenSearchDocumentStore from haystack_integrations.document_stores.opensearch.auth import AWSAuth from haystack_integrations.document_stores.opensearch.document_store import DEFAULT_MAX_CHUNK_BYTES -from opensearchpy.exceptions import RequestError @patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch") diff --git a/integrations/opensearch/tests/test_embedding_retriever.py b/integrations/opensearch/tests/test_embedding_retriever.py index e52a099c8..75c191946 100644 --- a/integrations/opensearch/tests/test_embedding_retriever.py +++ b/integrations/opensearch/tests/test_embedding_retriever.py @@ -6,6 +6,7 @@ import pytest from haystack.dataclasses import Document from haystack.document_stores.types import FilterPolicy + from haystack_integrations.components.retrievers.opensearch import OpenSearchEmbeddingRetriever from haystack_integrations.document_stores.opensearch import OpenSearchDocumentStore from haystack_integrations.document_stores.opensearch.document_store import DEFAULT_MAX_CHUNK_BYTES diff --git a/integrations/opensearch/tests/test_filters.py b/integrations/opensearch/tests/test_filters.py index d333dc584..966d0d363 100644 --- a/integrations/opensearch/tests/test_filters.py +++ b/integrations/opensearch/tests/test_filters.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 import pytest from haystack.errors import FilterError + from haystack_integrations.document_stores.opensearch.filters import _normalize_ranges, normalize_filters filters_data = [ From 13cdaab1049e83ef6d17095db4f5a950eb5f5ed9 Mon Sep 17 00:00:00 2001 From: Vladimir Blagojevic Date: Fri, 16 Aug 2024 12:38:57 +0100 Subject: [PATCH 21/25] Ruff update, don't ruff tests (#987) --- integrations/instructor_embedders/pyproject.toml | 4 ++-- .../instructor_embedders/tests/test_instructor_backend.py | 1 + .../tests/test_instructor_document_embedder.py | 1 + .../tests/test_instructor_text_embedder.py | 1 + 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/integrations/instructor_embedders/pyproject.toml b/integrations/instructor_embedders/pyproject.toml index 0543a9a88..017062a47 100644 --- a/integrations/instructor_embedders/pyproject.toml +++ b/integrations/instructor_embedders/pyproject.toml @@ -88,8 +88,8 @@ dependencies = ["black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243"] [tool.hatch.envs.lint.scripts] typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}" -style = ["ruff check {args:.}", "black --check --diff {args:.}"] -fmt = ["black {args:.}", "ruff --fix {args:.}", "style"] +style = ["ruff check {args:. --exclude tests/}", "black --check --diff {args:.}"] +fmt = ["black {args:.}", "ruff --fix {args:. --exclude tests/}", "style"] all = ["style", "typing"] [tool.coverage.run] diff --git a/integrations/instructor_embedders/tests/test_instructor_backend.py b/integrations/instructor_embedders/tests/test_instructor_backend.py index 85c1f012a..fdf6e44be 100644 --- a/integrations/instructor_embedders/tests/test_instructor_backend.py +++ b/integrations/instructor_embedders/tests/test_instructor_backend.py @@ -1,6 +1,7 @@ from unittest.mock import patch from haystack.utils.auth import Secret + from haystack_integrations.components.embedders.instructor_embedders.embedding_backend.instructor_backend import ( _InstructorEmbeddingBackendFactory, ) diff --git a/integrations/instructor_embedders/tests/test_instructor_document_embedder.py b/integrations/instructor_embedders/tests/test_instructor_document_embedder.py index 44f740679..f1676b411 100644 --- a/integrations/instructor_embedders/tests/test_instructor_document_embedder.py +++ b/integrations/instructor_embedders/tests/test_instructor_document_embedder.py @@ -4,6 +4,7 @@ import pytest from haystack import Document from haystack.utils import ComponentDevice, Secret + from haystack_integrations.components.embedders.instructor_embedders import InstructorDocumentEmbedder diff --git a/integrations/instructor_embedders/tests/test_instructor_text_embedder.py b/integrations/instructor_embedders/tests/test_instructor_text_embedder.py index 55022f1ec..68e3e5d62 100644 --- a/integrations/instructor_embedders/tests/test_instructor_text_embedder.py +++ b/integrations/instructor_embedders/tests/test_instructor_text_embedder.py @@ -3,6 +3,7 @@ import numpy as np import pytest from haystack.utils import ComponentDevice, Secret + from haystack_integrations.components.embedders.instructor_embedders import InstructorTextEmbedder From 55c65af5ac338f0678537125d6fd0e92dfa689b6 Mon Sep 17 00:00:00 2001 From: Madeesh Kannan Date: Fri, 16 Aug 2024 14:48:23 +0200 Subject: [PATCH 22/25] refactor: Remove usage of deprecated `ChatMessage.to_openai_format` (#1001) --- .../src/haystack_integrations/tracing/langfuse/tracer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/integrations/langfuse/src/haystack_integrations/tracing/langfuse/tracer.py b/integrations/langfuse/src/haystack_integrations/tracing/langfuse/tracer.py index 4bf0da2f8..7d141c08c 100644 --- a/integrations/langfuse/src/haystack_integrations/tracing/langfuse/tracer.py +++ b/integrations/langfuse/src/haystack_integrations/tracing/langfuse/tracer.py @@ -2,6 +2,7 @@ import os from typing import Any, Dict, Iterator, Optional, Union +from haystack.components.generators.openai_utils import _convert_message_to_openai_format from haystack.dataclasses import ChatMessage from haystack.tracing import Span, Tracer, tracer from haystack.tracing import utils as tracing_utils @@ -51,14 +52,14 @@ def set_content_tag(self, key: str, value: Any) -> None: return if key.endswith(".input"): if "messages" in value: - messages = [m.to_openai_format() for m in value["messages"]] + messages = [_convert_message_to_openai_format(m) for m in value["messages"]] self._span.update(input=messages) else: self._span.update(input=value) elif key.endswith(".output"): if "replies" in value: if all(isinstance(r, ChatMessage) for r in value["replies"]): - replies = [m.to_openai_format() for m in value["replies"]] + replies = [_convert_message_to_openai_format(m) for m in value["replies"]] else: replies = value["replies"] self._span.update(output=replies) From e72b455efb00dc710a84ab5088c9c4b79ab08347 Mon Sep 17 00:00:00 2001 From: "David S. Batista" Date: Mon, 19 Aug 2024 11:35:15 +0200 Subject: [PATCH 23/25] fix: normalising ChatGenerators output (#973) * initial import * fixing tests * cleaning * all under 'usage' field * removing exception * Update integrations/amazon_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/chat/chat_generator.py Co-authored-by: Stefano Fiorucci * reverting * fixing old test * linting --------- Co-authored-by: Stefano Fiorucci --- .../generators/amazon_bedrock/chat/adapters.py | 2 +- .../amazon_bedrock/chat/chat_generator.py | 15 ++++++++++++--- .../tests/test_chat_generator.py | 18 ++++++++++-------- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/integrations/amazon_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/chat/adapters.py b/integrations/amazon_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/chat/adapters.py index 1f0430810..87bd61b53 100644 --- a/integrations/amazon_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/chat/adapters.py +++ b/integrations/amazon_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/chat/adapters.py @@ -341,7 +341,7 @@ def __init__(self, truncate: Optional[bool], generation_kwargs: Dict[str, Any]): # b) we can use apply_chat_template with the template above to delineate ChatMessages # Mistral models are gated on HF Hub. If no HF_TOKEN is found we use a non-gated alternative tokenizer model. tokenizer: PreTrainedTokenizer - if os.environ.get("HF_TOKEN"): + if os.environ.get("HF_TOKEN") or os.environ.get("HF_API_TOKEN"): tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.1") else: tokenizer = AutoTokenizer.from_pretrained("NousResearch/Llama-2-7b-chat-hf") diff --git a/integrations/amazon_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/chat/chat_generator.py b/integrations/amazon_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/chat/chat_generator.py index 5fa9e0b8a..988452a97 100644 --- a/integrations/amazon_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/chat/chat_generator.py +++ b/integrations/amazon_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/chat/chat_generator.py @@ -208,9 +208,18 @@ def run( # rename the meta key to be inline with OpenAI meta output keys for response in replies: - if response.meta is not None and "usage" in response.meta: - response.meta["usage"]["prompt_tokens"] = response.meta["usage"].pop("input_tokens") - response.meta["usage"]["completion_tokens"] = response.meta["usage"].pop("output_tokens") + if response.meta: + if "usage" in response.meta: + if "input_tokens" in response.meta["usage"]: + response.meta["usage"]["prompt_tokens"] = response.meta["usage"].pop("input_tokens") + if "output_tokens" in response.meta["usage"]: + response.meta["usage"]["completion_tokens"] = response.meta["usage"].pop("output_tokens") + else: + response.meta["usage"] = {} + if "prompt_token_count" in response.meta: + response.meta["usage"]["prompt_tokens"] = response.meta.pop("prompt_token_count") + if "generation_token_count" in response.meta: + response.meta["usage"]["completion_tokens"] = response.meta.pop("generation_token_count") return {"replies": replies} diff --git a/integrations/amazon_bedrock/tests/test_chat_generator.py b/integrations/amazon_bedrock/tests/test_chat_generator.py index 64e9ce2ef..98f20fc2d 100644 --- a/integrations/amazon_bedrock/tests/test_chat_generator.py +++ b/integrations/amazon_bedrock/tests/test_chat_generator.py @@ -228,14 +228,12 @@ def test_long_prompt_is_not_truncated_when_truncate_false(mock_boto3_session): content="Some text", role=ChatRole.ASSISTANT, name=None, - meta=[ - { - "model": "claude-3-sonnet-20240229", - "index": 0, - "finish_reason": "end_turn", - "usage": {"prompt_tokens": 16, "completion_tokens": 55}, - } - ], + meta={ + "model": "claude-3-sonnet-20240229", + "index": 0, + "finish_reason": "end_turn", + "usage": {"prompt_tokens": 16, "completion_tokens": 55}, + }, ) ] ) @@ -489,6 +487,10 @@ def test_default_inference_params(self, model_name, chat_messages): assert "paris" in first_reply.content.lower(), "First reply does not contain 'paris'" assert first_reply.meta, "First reply has no metadata" + if first_reply.meta and "usage" in first_reply.meta: + assert "prompt_tokens" in first_reply.meta["usage"] + assert "completion_tokens" in first_reply.meta["usage"] + @pytest.mark.parametrize("model_name", MODELS_TO_TEST) @pytest.mark.integration def test_default_inference_with_streaming(self, model_name, chat_messages): From 36c5401f2c8a3ae4465afcc00702e0d0f5cacdc7 Mon Sep 17 00:00:00 2001 From: HaystackBot Date: Mon, 19 Aug 2024 10:28:44 +0000 Subject: [PATCH 24/25] Update the changelog --- integrations/amazon_bedrock/CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/integrations/amazon_bedrock/CHANGELOG.md b/integrations/amazon_bedrock/CHANGELOG.md index d347c08d9..631978058 100644 --- a/integrations/amazon_bedrock/CHANGELOG.md +++ b/integrations/amazon_bedrock/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## [integrations/amazon_bedrock-v1.0.1] - 2024-08-19 + +### 🚀 Features + +- Make truncation optional for bedrock chat generator (#967) + +### 🐛 Bug Fixes + +- Normalising ChatGenerators output (#973) + ## [integrations/amazon_bedrock-v1.0.0] - 2024-08-12 ### 🚜 Refactor From 39decab96a98fe7a358d34796af22469b2084dbb Mon Sep 17 00:00:00 2001 From: Madeesh Kannan Date: Mon, 19 Aug 2024 15:33:48 +0200 Subject: [PATCH 25/25] refactor: Remove usage of deprecated `ChatMessage.to_openai_format` (#1007) --- .../amazon_bedrock/chat/adapters.py | 20 ++++--------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/integrations/amazon_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/chat/adapters.py b/integrations/amazon_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/chat/adapters.py index 87bd61b53..56eefdf09 100644 --- a/integrations/amazon_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/chat/adapters.py +++ b/integrations/amazon_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/chat/adapters.py @@ -5,6 +5,7 @@ from typing import Any, Callable, ClassVar, Dict, List, Optional from botocore.eventstream import EventStream +from haystack.components.generators.openai_utils import _convert_message_to_openai_format from haystack.dataclasses import ChatMessage, ChatRole, StreamingChunk from transformers import AutoTokenizer, PreTrainedTokenizer @@ -389,27 +390,14 @@ def prepare_chat_messages(self, messages: List[ChatMessage]) -> str: # default is https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1/blob/main/tokenizer_config.json # but we'll use our custom chat template prepared_prompt: str = self.prompt_handler.tokenizer.apply_chat_template( - conversation=[self.to_openai_format(m) for m in messages], tokenize=False, chat_template=self.chat_template + conversation=[_convert_message_to_openai_format(m) for m in messages], + tokenize=False, + chat_template=self.chat_template, ) if self.truncate: prepared_prompt = self._ensure_token_limit(prepared_prompt) return prepared_prompt - def to_openai_format(self, m: ChatMessage) -> Dict[str, Any]: - """ - Convert the message to the format expected by OpenAI's Chat API. - See the [API reference](https://platform.openai.com/docs/api-reference/chat/create) for details. - - :returns: A dictionary with the following key: - - `role` - - `content` - - `name` (optional) - """ - msg = {"role": m.role.value, "content": m.content} - if m.name: - msg["name"] = m.name - return msg - def check_prompt(self, prompt: str) -> Dict[str, Any]: """ Checks the prompt length and resizes it if necessary. If the prompt is too long, it will be truncated.