diff --git a/.github/workflows/google_vertex.yml b/.github/workflows/google_vertex.yml index 78ba5694b..34c0cf07c 100644 --- a/.github/workflows/google_vertex.yml +++ b/.github/workflows/google_vertex.yml @@ -30,7 +30,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python-version: ["3.9", "3.10"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - name: Support longpaths diff --git a/.github/workflows/weaviate.yml b/.github/workflows/weaviate.yml index 5e29eafe7..06a4bc289 100644 --- a/.github/workflows/weaviate.yml +++ b/.github/workflows/weaviate.yml @@ -44,6 +44,7 @@ jobs: run: pip install --upgrade hatch - name: Lint + if: matrix.python-version == '3.9' && runner.os == 'Linux' run: hatch run lint:all - name: Run Weaviate container diff --git a/README.md b/README.md index c4178184b..7ba853d62 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Please check out our [Contribution Guidelines](CONTRIBUTING.md) for all the deta | [mistral-haystack](integrations/mistral/) | Embedder, Generator | [![PyPI - Version](https://img.shields.io/pypi/v/mistral-haystack.svg)](https://pypi.org/project/mistral-haystack) | [![Test / mistral](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/mistral.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/mistral.yml) | | [mongodb-atlas-haystack](integrations/mongodb_atlas/) | Document Store | [![PyPI - Version](https://img.shields.io/pypi/v/mongodb-atlas-haystack.svg?color=orange)](https://pypi.org/project/mongodb-atlas-haystack) | [![Test / mongodb-atlas](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/mongodb_atlas.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/mongodb_atlas.yml) | | [nvidia-haystack](integrations/nvidia/) | Generator | [![PyPI - Version](https://img.shields.io/pypi/v/nvidia-haystack.svg?color=orange)](https://pypi.org/project/nvidia-haystack) | [![Test / nvidia](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/nvidia.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/nvidia.yml) | -| [ollama-haystack](integrations/ollama/) | Generator | [![PyPI - Version](https://img.shields.io/pypi/v/ollama-haystack.svg?color=orange)](https://pypi.org/project/ollama-haystack) | [![Test / ollama](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/ollama.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/ollama.yml) | +| [ollama-haystack](integrations/ollama/) | Embedder, Generator | [![PyPI - Version](https://img.shields.io/pypi/v/ollama-haystack.svg?color=orange)](https://pypi.org/project/ollama-haystack) | [![Test / ollama](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/ollama.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/ollama.yml) | | [opensearch-haystack](integrations/opensearch/) | Document Store | [![PyPI - Version](https://img.shields.io/pypi/v/opensearch-haystack.svg)](https://pypi.org/project/opensearch-haystack) | [![Test / opensearch](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/opensearch.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/opensearch.yml) | | [optimum-haystack](integrations/optimum/) | Embedder | [![PyPI - Version](https://img.shields.io/pypi/v/optimum-haystack.svg)](https://pypi.org/project/optimum-haystack) | [![Test / optimum](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/optimum.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/optimum.yml) | | [pinecone-haystack](integrations/pinecone/) | Document Store | [![PyPI - Version](https://img.shields.io/pypi/v/pinecone-haystack.svg?color=orange)](https://pypi.org/project/pinecone-haystack) | [![Test / pinecone](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/pinecone.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/pinecone.yml) | diff --git a/integrations/astra/CHANGELOG.md b/integrations/astra/CHANGELOG.md index 55c22f540..79bb9e35d 100644 --- a/integrations/astra/CHANGELOG.md +++ b/integrations/astra/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## [integrations/astra-v0.9.3] - 2024-09-12 + +### ๐Ÿ› Bug Fixes + +- Astra DB, improved warnings and guidance about indexing-related mismatches (#932) +- AstraDocumentStore filter by id (#1053) + +### ๐Ÿงช Testing + +- Do not retry tests in `hatch run test` command (#954) + ## [integrations/astra-v0.9.2] - 2024-07-22 ## [integrations/astra-v0.9.1] - 2024-07-15 diff --git a/integrations/astra/src/haystack_integrations/document_stores/astra/filters.py b/integrations/astra/src/haystack_integrations/document_stores/astra/filters.py index 61f3e5402..340e95ba9 100644 --- a/integrations/astra/src/haystack_integrations/document_stores/astra/filters.py +++ b/integrations/astra/src/haystack_integrations/document_stores/astra/filters.py @@ -30,8 +30,6 @@ def _convert_filters(filters: Optional[Dict[str, Any]] = None) -> Optional[Dict[ if key in {"$and", "$or"}: filter_statements[key] = value else: - if key == "id": - filter_statements[key] = {"_id": value} if key != "$in" and isinstance(value, list): filter_statements[key] = {"$in": value} elif isinstance(value, pd.DataFrame): @@ -45,6 +43,8 @@ def _convert_filters(filters: Optional[Dict[str, Any]] = None) -> Optional[Dict[ filter_statements[key] = converted else: filter_statements[key] = value + if key == "id": + filter_statements["_id"] = filter_statements.pop("id") return filter_statements diff --git a/integrations/astra/tests/test_document_store.py b/integrations/astra/tests/test_document_store.py index df181ad8c..c4d1b6347 100644 --- a/integrations/astra/tests/test_document_store.py +++ b/integrations/astra/tests/test_document_store.py @@ -200,6 +200,12 @@ def test_filter_documents_nested_filters(self, document_store, filterable_docs): ], ) + def test_filter_documents_by_id(self, document_store): + docs = [Document(id="1", content="test doc 1"), Document(id="2", content="test doc 2")] + document_store.write_documents(docs) + result = document_store.filter_documents(filters={"field": "id", "operator": "==", "value": "1"}) + self.assert_documents_are_equal(result, [docs[0]]) + @pytest.mark.skip(reason="Unsupported filter operator not.") def test_not_operator(self, document_store, filterable_docs): pass diff --git a/integrations/chroma/src/haystack_integrations/document_stores/chroma/document_store.py b/integrations/chroma/src/haystack_integrations/document_stores/chroma/document_store.py index 3ea84780f..3353ed5aa 100644 --- a/integrations/chroma/src/haystack_integrations/document_stores/chroma/document_store.py +++ b/integrations/chroma/src/haystack_integrations/document_stores/chroma/document_store.py @@ -6,7 +6,6 @@ from typing import Any, Dict, List, Literal, Optional, Tuple import chromadb -import numpy as np from chromadb.api.types import GetResult, QueryResult, validate_where, validate_where_document from haystack import default_from_dict, default_to_dict from haystack.dataclasses import Document @@ -453,7 +452,7 @@ def _query_result_to_documents(result: QueryResult) -> List[List[Document]]: for j in range(len(answers)): document_dict: Dict[str, Any] = { "id": result["ids"][i][j], - "content": documents[i][j], + "content": answers[j], } # prepare metadata @@ -465,7 +464,7 @@ def _query_result_to_documents(result: QueryResult) -> List[List[Document]]: pass if embeddings := result.get("embeddings"): - document_dict["embedding"] = np.array(embeddings[i][j]) + document_dict["embedding"] = embeddings[i][j] if distances := result.get("distances"): document_dict["score"] = distances[i][j] diff --git a/integrations/chroma/tests/test_document_store.py b/integrations/chroma/tests/test_document_store.py index d4b6ed272..5a7e12b3d 100644 --- a/integrations/chroma/tests/test_document_store.py +++ b/integrations/chroma/tests/test_document_store.py @@ -106,7 +106,12 @@ def test_search(self): # Assertions to verify correctness assert len(result) == 1 - assert result[0][0].content == "Third document" + doc = result[0][0] + assert doc.content == "Third document" + assert doc.meta == {"author": "Author2"} + assert doc.embedding + assert isinstance(doc.embedding, list) + assert all(isinstance(el, float) for el in doc.embedding) def test_write_documents_unsupported_meta_values(self, document_store: ChromaDocumentStore): """ diff --git a/integrations/elasticsearch/CHANGELOG.md b/integrations/elasticsearch/CHANGELOG.md index a825234bc..5d2b66470 100644 --- a/integrations/elasticsearch/CHANGELOG.md +++ b/integrations/elasticsearch/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [unreleased] +## [integrations/elasticsearch-v1.0.0] - 2024-09-12 ### ๐Ÿš€ Features @@ -11,10 +11,15 @@ - `ElasticSearch` - Fallback to default filter policy when deserializing retrievers without the init parameter (#898) +### ๐Ÿงช Testing + +- Do not retry tests in `hatch run test` command (#954) + ### โš™๏ธ Miscellaneous Tasks - Retry tests to reduce flakyness (#836) - Update ruff invocation to include check parameter (#853) +- ElasticSearch - remove legacy filters elasticsearch (#1078) ## [integrations/elasticsearch-v0.5.0] - 2024-05-24 diff --git a/integrations/elasticsearch/src/haystack_integrations/document_stores/elasticsearch/document_store.py b/integrations/elasticsearch/src/haystack_integrations/document_stores/elasticsearch/document_store.py index 11016e3fc..734e2d2b8 100644 --- a/integrations/elasticsearch/src/haystack_integrations/document_stores/elasticsearch/document_store.py +++ b/integrations/elasticsearch/src/haystack_integrations/document_stores/elasticsearch/document_store.py @@ -12,7 +12,6 @@ from haystack.dataclasses import Document from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError from haystack.document_stores.types import DuplicatePolicy -from haystack.utils.filters import convert from haystack.version import __version__ as haystack_version from elasticsearch import Elasticsearch, helpers # type: ignore[import-not-found] @@ -224,7 +223,8 @@ def filter_documents(self, filters: Optional[Dict[str, Any]] = None) -> List[Doc :returns: List of `Document`s that match the filters. """ if filters and "operator" not in filters and "conditions" not in filters: - filters = convert(filters) + msg = "Invalid filter syntax. See https://docs.haystack.deepset.ai/docs/metadata-filtering for details." + raise ValueError(msg) query = {"bool": {"filter": _normalize_filters(filters)}} if filters else None documents = self._search_documents(query=query) diff --git a/integrations/google_vertex/CHANGELOG.md b/integrations/google_vertex/CHANGELOG.md new file mode 100644 index 000000000..17a730b60 --- /dev/null +++ b/integrations/google_vertex/CHANGELOG.md @@ -0,0 +1,58 @@ +# Changelog + +## [unreleased] + +### ๐Ÿš€ Features + +- Enable streaming for VertexAIGeminiChatGenerator (#1014) +- Add tests for VertexAIGeminiGenerator and enable streaming (#1012) + +### ๐Ÿ› Bug Fixes + +- Remove the use of deprecated gemini models (#1032) +- Chat roles for model responses in chat generators (#1030) + +### ๐Ÿงช Testing + +- Do not retry tests in `hatch run test` command (#954) +- Add tests for VertexAIChatGeminiGenerator and migrate from preview package in vertexai (#1042) + +### โš™๏ธ Miscellaneous Tasks + +- Retry tests to reduce flakyness (#836) +- Update ruff invocation to include check parameter (#853) + +## [integrations/google_vertex-v1.1.0] - 2024-03-28 + +## [integrations/google_vertex-v1.0.0] - 2024-03-27 + +### ๐Ÿ› Bug Fixes + +- Fix order of API docs (#447) + +This PR will also push the docs to Readme + +### ๐Ÿ“š Documentation + +- Update category slug (#442) +- Review google vertex integration (#535) +- Small consistency improvements (#536) +- Disable-class-def (#556) + +### Google_vertex + +- Create api docs (#355) + +## [integrations/google_vertex-v0.2.0] - 2024-01-26 + +## [integrations/google_vertex-v0.1.0] - 2024-01-03 + +### ๐Ÿ› Bug Fixes + +- The default model of VertexAIImagegenerator (#158) + +### โš™๏ธ Miscellaneous Tasks + +- Replace - with _ (#114) + + diff --git a/integrations/google_vertex/pyproject.toml b/integrations/google_vertex/pyproject.toml index 747bbecbf..71158f712 100644 --- a/integrations/google_vertex/pyproject.toml +++ b/integrations/google_vertex/pyproject.toml @@ -22,7 +22,12 @@ classifiers = [ "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ] -dependencies = ["haystack-ai", "google-cloud-aiplatform>=1.38", "pyarrow>3"] +dependencies = [ + "haystack-ai", + "google-cloud-aiplatform>=1.38", + "pyarrow>3", + "protobuf<5.28", +] [project.urls] Documentation = "https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/google_vertex#readme" diff --git a/integrations/google_vertex/src/haystack_integrations/components/generators/google_vertex/chat/gemini.py b/integrations/google_vertex/src/haystack_integrations/components/generators/google_vertex/chat/gemini.py index e5ca1166d..e693c10f4 100644 --- a/integrations/google_vertex/src/haystack_integrations/components/generators/google_vertex/chat/gemini.py +++ b/integrations/google_vertex/src/haystack_integrations/components/generators/google_vertex/chat/gemini.py @@ -8,7 +8,7 @@ from haystack.dataclasses.chat_message import ChatMessage, ChatRole from haystack.utils import deserialize_callable, serialize_callable from vertexai import init as vertexai_init -from vertexai.preview.generative_models import ( +from vertexai.generative_models import ( Content, GenerationConfig, GenerationResponse, @@ -67,14 +67,14 @@ def __init__( :param location: The default location to use when making API calls, if not set uses us-central-1. Defaults to None. :param generation_config: Configuration for the generation process. - See the [GenerationConfig documentation](https://cloud.google.com/python/docs/reference/aiplatform/latest/vertexai.preview.generative_models.GenerationConfig + See the [GenerationConfig documentation](https://cloud.google.com/python/docs/reference/aiplatform/latest/vertexai.generative_models.GenerationConfig for a list of supported arguments. :param safety_settings: Safety settings to use when generating content. See the documentation - for [HarmBlockThreshold](https://cloud.google.com/python/docs/reference/aiplatform/latest/vertexai.preview.generative_models.HarmBlockThreshold) - and [HarmCategory](https://cloud.google.com/python/docs/reference/aiplatform/latest/vertexai.preview.generative_models.HarmCategory) + for [HarmBlockThreshold](https://cloud.google.com/python/docs/reference/aiplatform/latest/vertexai.generative_models.HarmBlockThreshold) + and [HarmCategory](https://cloud.google.com/python/docs/reference/aiplatform/latest/vertexai.generative_models.HarmCategory) for more details. :param tools: List of tools to use when generating content. See the documentation for - [Tool](https://cloud.google.com/python/docs/reference/aiplatform/latest/vertexai.preview.generative_models.Tool) + [Tool](https://cloud.google.com/python/docs/reference/aiplatform/latest/vertexai.generative_models.Tool) the list of supported arguments. :param streaming_callback: A callback function that is called when a new token is received from the stream. The callback function accepts StreamingChunk as an argument. diff --git a/integrations/google_vertex/src/haystack_integrations/components/generators/google_vertex/gemini.py b/integrations/google_vertex/src/haystack_integrations/components/generators/google_vertex/gemini.py index 11592671f..7394211bf 100644 --- a/integrations/google_vertex/src/haystack_integrations/components/generators/google_vertex/gemini.py +++ b/integrations/google_vertex/src/haystack_integrations/components/generators/google_vertex/gemini.py @@ -70,7 +70,7 @@ def __init__( :param model: Name of the model to use. For available models, see https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models. :param location: The default location to use when making API calls, if not set uses us-central-1. :param generation_config: The generation config to use. - Can either be a [`GenerationConfig`](https://cloud.google.com/python/docs/reference/aiplatform/latest/vertexai.preview.generative_models.GenerationConfig) + Can either be a [`GenerationConfig`](https://cloud.google.com/python/docs/reference/aiplatform/latest/vertexai.generative_models.GenerationConfig) object or a dictionary of parameters. Accepted fields are: - temperature @@ -80,11 +80,11 @@ def __init__( - max_output_tokens - stop_sequences :param safety_settings: The safety settings to use. See the documentation - for [HarmBlockThreshold](https://cloud.google.com/python/docs/reference/aiplatform/latest/vertexai.preview.generative_models.HarmBlockThreshold) - and [HarmCategory](https://cloud.google.com/python/docs/reference/aiplatform/latest/vertexai.preview.generative_models.HarmCategory) + for [HarmBlockThreshold](https://cloud.google.com/python/docs/reference/aiplatform/latest/vertexai.generative_models.HarmBlockThreshold) + and [HarmCategory](https://cloud.google.com/python/docs/reference/aiplatform/latest/vertexai.generative_models.HarmCategory) for more details. :param tools: List of tools to use when generating content. See the documentation for - [Tool](https://cloud.google.com/python/docs/reference/aiplatform/latest/vertexai.preview.generative_models.Tool) + [Tool](https://cloud.google.com/python/docs/reference/aiplatform/latest/vertexai.generative_models.Tool) the list of supported arguments. :param streaming_callback: A callback function that is called when a new token is received from the stream. The callback function accepts StreamingChunk as an argument. diff --git a/integrations/google_vertex/src/haystack_integrations/components/generators/google_vertex/image_generator.py b/integrations/google_vertex/src/haystack_integrations/components/generators/google_vertex/image_generator.py index ae8c4892f..0534a20f2 100644 --- a/integrations/google_vertex/src/haystack_integrations/components/generators/google_vertex/image_generator.py +++ b/integrations/google_vertex/src/haystack_integrations/components/generators/google_vertex/image_generator.py @@ -5,7 +5,7 @@ from haystack.core.component import component from haystack.core.serialization import default_from_dict, default_to_dict from haystack.dataclasses.byte_stream import ByteStream -from vertexai.preview.vision_models import ImageGenerationModel +from vertexai.vision_models import ImageGenerationModel logger = logging.getLogger(__name__) diff --git a/integrations/google_vertex/tests/chat/test_gemini.py b/integrations/google_vertex/tests/chat/test_gemini.py new file mode 100644 index 000000000..a1564b9f2 --- /dev/null +++ b/integrations/google_vertex/tests/chat/test_gemini.py @@ -0,0 +1,295 @@ +from unittest.mock import MagicMock, Mock, patch + +import pytest +from haystack import Pipeline +from haystack.components.builders import ChatPromptBuilder +from haystack.dataclasses import ChatMessage, StreamingChunk +from vertexai.generative_models import ( + Content, + FunctionDeclaration, + GenerationConfig, + GenerationResponse, + HarmBlockThreshold, + HarmCategory, + Part, + Tool, +) + +from haystack_integrations.components.generators.google_vertex import VertexAIGeminiChatGenerator + +GET_CURRENT_WEATHER_FUNC = FunctionDeclaration( + name="get_current_weather", + description="Get the current weather in a given location", + parameters={ + "type_": "OBJECT", + "properties": { + "location": {"type_": "STRING", "description": "The city and state, e.g. San Francisco, CA"}, + "unit": { + "type_": "STRING", + "enum": [ + "celsius", + "fahrenheit", + ], + }, + }, + "required": ["location"], + }, +) + + +@pytest.fixture +def chat_messages(): + return [ + ChatMessage.from_system("You are a helpful assistant"), + ChatMessage.from_user("What's the capital of France"), + ] + + +@patch("haystack_integrations.components.generators.google_vertex.chat.gemini.vertexai_init") +@patch("haystack_integrations.components.generators.google_vertex.chat.gemini.GenerativeModel") +def test_init(mock_vertexai_init, _mock_generative_model): + + generation_config = GenerationConfig( + candidate_count=1, + stop_sequences=["stop"], + max_output_tokens=10, + temperature=0.5, + top_p=0.5, + top_k=0.5, + ) + safety_settings = {HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_ONLY_HIGH} + + tool = Tool(function_declarations=[GET_CURRENT_WEATHER_FUNC]) + + gemini = VertexAIGeminiChatGenerator( + project_id="TestID123", + location="TestLocation", + generation_config=generation_config, + safety_settings=safety_settings, + tools=[tool], + ) + mock_vertexai_init.assert_called() + assert gemini._model_name == "gemini-1.5-flash" + assert gemini._generation_config == generation_config + assert gemini._safety_settings == safety_settings + assert gemini._tools == [tool] + + +@patch("haystack_integrations.components.generators.google_vertex.chat.gemini.vertexai_init") +@patch("haystack_integrations.components.generators.google_vertex.chat.gemini.GenerativeModel") +def test_to_dict(_mock_vertexai_init, _mock_generative_model): + + gemini = VertexAIGeminiChatGenerator( + project_id="TestID123", + ) + assert gemini.to_dict() == { + "type": "haystack_integrations.components.generators.google_vertex.chat.gemini.VertexAIGeminiChatGenerator", + "init_parameters": { + "model": "gemini-1.5-flash", + "project_id": "TestID123", + "location": None, + "generation_config": None, + "safety_settings": None, + "streaming_callback": None, + "tools": None, + }, + } + + +@patch("haystack_integrations.components.generators.google_vertex.chat.gemini.vertexai_init") +@patch("haystack_integrations.components.generators.google_vertex.chat.gemini.GenerativeModel") +def test_to_dict_with_params(_mock_vertexai_init, _mock_generative_model): + generation_config = GenerationConfig( + candidate_count=1, + stop_sequences=["stop"], + max_output_tokens=10, + temperature=0.5, + top_p=0.5, + top_k=2, + ) + safety_settings = {HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_ONLY_HIGH} + + tool = Tool(function_declarations=[GET_CURRENT_WEATHER_FUNC]) + + gemini = VertexAIGeminiChatGenerator( + project_id="TestID123", + generation_config=generation_config, + safety_settings=safety_settings, + tools=[tool], + ) + + assert gemini.to_dict() == { + "type": "haystack_integrations.components.generators.google_vertex.chat.gemini.VertexAIGeminiChatGenerator", + "init_parameters": { + "model": "gemini-1.5-flash", + "project_id": "TestID123", + "location": None, + "generation_config": { + "temperature": 0.5, + "top_p": 0.5, + "top_k": 2.0, + "candidate_count": 1, + "max_output_tokens": 10, + "stop_sequences": ["stop"], + }, + "safety_settings": {HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_ONLY_HIGH}, + "streaming_callback": None, + "tools": [ + { + "function_declarations": [ + { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type_": "OBJECT", + "properties": { + "location": { + "type_": "STRING", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type_": "STRING", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + } + ] + } + ], + }, + } + + +@patch("haystack_integrations.components.generators.google_vertex.chat.gemini.vertexai_init") +@patch("haystack_integrations.components.generators.google_vertex.chat.gemini.GenerativeModel") +def test_from_dict(_mock_vertexai_init, _mock_generative_model): + gemini = VertexAIGeminiChatGenerator.from_dict( + { + "type": "haystack_integrations.components.generators.google_vertex.chat.gemini.VertexAIGeminiChatGenerator", + "init_parameters": { + "project_id": "TestID123", + "model": "gemini-1.5-flash", + "generation_config": None, + "safety_settings": None, + "tools": None, + "streaming_callback": None, + }, + } + ) + + assert gemini._model_name == "gemini-1.5-flash" + assert gemini._project_id == "TestID123" + assert gemini._safety_settings is None + assert gemini._tools is None + assert gemini._generation_config is None + + +@patch("haystack_integrations.components.generators.google_vertex.chat.gemini.vertexai_init") +@patch("haystack_integrations.components.generators.google_vertex.chat.gemini.GenerativeModel") +def test_from_dict_with_param(_mock_vertexai_init, _mock_generative_model): + gemini = VertexAIGeminiChatGenerator.from_dict( + { + "type": "haystack_integrations.components.generators.google_vertex.chat.gemini.VertexAIGeminiChatGenerator", + "init_parameters": { + "project_id": "TestID123", + "model": "gemini-1.5-flash", + "generation_config": { + "temperature": 0.5, + "top_p": 0.5, + "top_k": 0.5, + "candidate_count": 1, + "max_output_tokens": 10, + "stop_sequences": ["stop"], + }, + "safety_settings": {HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_ONLY_HIGH}, + "tools": [ + { + "function_declarations": [ + { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type_": "OBJECT", + "properties": { + "location": { + "type_": "STRING", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type_": "STRING", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + } + ] + } + ], + "streaming_callback": None, + }, + } + ) + + assert gemini._model_name == "gemini-1.5-flash" + assert gemini._project_id == "TestID123" + assert gemini._safety_settings == {HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_ONLY_HIGH} + assert repr(gemini._tools) == repr([Tool(function_declarations=[GET_CURRENT_WEATHER_FUNC])]) + assert isinstance(gemini._generation_config, GenerationConfig) + + +@patch("haystack_integrations.components.generators.google_vertex.chat.gemini.GenerativeModel") +def test_run(mock_generative_model): + mock_model = Mock() + mock_candidate = Mock(content=Content(parts=[Part.from_text("This is a generated response.")], role="model")) + mock_response = MagicMock(spec=GenerationResponse, candidates=[mock_candidate]) + + mock_model.send_message.return_value = mock_response + mock_model.start_chat.return_value = mock_model + mock_generative_model.return_value = mock_model + + messages = [ + ChatMessage.from_system("You are a helpful assistant"), + ChatMessage.from_user("What's the capital of France?"), + ] + gemini = VertexAIGeminiChatGenerator(project_id="TestID123", location=None) + gemini.run(messages=messages) + + mock_model.send_message.assert_called_once() + + +@patch("haystack_integrations.components.generators.google_vertex.chat.gemini.GenerativeModel") +def test_run_with_streaming_callback(mock_generative_model): + mock_model = Mock() + mock_responses = iter( + [MagicMock(spec=GenerationResponse, text="First part"), MagicMock(spec=GenerationResponse, text="Second part")] + ) + + mock_model.send_message.return_value = mock_responses + mock_model.start_chat.return_value = mock_model + mock_generative_model.return_value = mock_model + + streaming_callback_called = [] + + def streaming_callback(chunk: StreamingChunk) -> None: + streaming_callback_called.append(chunk.content) + + gemini = VertexAIGeminiChatGenerator(project_id="TestID123", location=None, streaming_callback=streaming_callback) + messages = [ + ChatMessage.from_system("You are a helpful assistant"), + ChatMessage.from_user("What's the capital of France?"), + ] + gemini.run(messages=messages) + + mock_model.send_message.assert_called_once() + assert streaming_callback_called == ["First part", "Second part"] + + +def test_serialization_deserialization_pipeline(): + + pipeline = Pipeline() + template = [ChatMessage.from_user("Translate to {{ target_language }}. Context: {{ snippet }}; Translation:")] + pipeline.add_component("prompt_builder", ChatPromptBuilder(template=template)) + pipeline.add_component("gemini", VertexAIGeminiChatGenerator(project_id="TestID123")) + pipeline.connect("prompt_builder.prompt", "gemini.messages") + + pipeline_dict = pipeline.to_dict() + + new_pipeline = Pipeline.from_dict(pipeline_dict) + assert new_pipeline == pipeline diff --git a/integrations/google_vertex/tests/test_gemini.py b/integrations/google_vertex/tests/test_gemini.py index 8d08e0859..bb96ec409 100644 --- a/integrations/google_vertex/tests/test_gemini.py +++ b/integrations/google_vertex/tests/test_gemini.py @@ -1,7 +1,9 @@ from unittest.mock import MagicMock, Mock, patch +from haystack import Pipeline +from haystack.components.builders import PromptBuilder from haystack.dataclasses import StreamingChunk -from vertexai.preview.generative_models import ( +from vertexai.generative_models import ( FunctionDeclaration, GenerationConfig, HarmBlockThreshold, @@ -191,18 +193,18 @@ def test_from_dict_with_param(_mock_vertexai_init, _mock_generative_model): "function_declarations": [ { "name": "get_current_weather", - "description": "Get the current weather in a given location", "parameters": { "type_": "OBJECT", "properties": { + "unit": {"type_": "STRING", "enum": ["celsius", "fahrenheit"]}, "location": { "type_": "STRING", "description": "The city and state, e.g. San Francisco, CA", }, - "unit": {"type_": "STRING", "enum": ["celsius", "fahrenheit"]}, }, "required": ["location"], }, + "description": "Get the current weather in a given location", } ] } @@ -254,3 +256,20 @@ def streaming_callback(_chunk: StreamingChunk) -> None: gemini = VertexAIGeminiGenerator(model="gemini-pro", project_id="TestID123", streaming_callback=streaming_callback) gemini.run(["Come on, stream!"]) assert streaming_callback_called + + +def test_serialization_deserialization_pipeline(): + template = """ + Answer the following questions: + 1. What is the weather like today? + """ + pipeline = Pipeline() + + pipeline.add_component("prompt_builder", PromptBuilder(template=template)) + pipeline.add_component("gemini", VertexAIGeminiGenerator(project_id="TestID123")) + pipeline.connect("prompt_builder", "gemini") + + pipeline_dict = pipeline.to_dict() + + new_pipeline = Pipeline.from_dict(pipeline_dict) + assert new_pipeline == pipeline diff --git a/integrations/google_vertex/tests/test_image_generator.py b/integrations/google_vertex/tests/test_image_generator.py index 42cc0a0a3..6cd42a11c 100644 --- a/integrations/google_vertex/tests/test_image_generator.py +++ b/integrations/google_vertex/tests/test_image_generator.py @@ -1,6 +1,6 @@ from unittest.mock import Mock, patch -from vertexai.preview.vision_models import ImageGenerationResponse +from vertexai.vision_models import ImageGenerationResponse from haystack_integrations.components.generators.google_vertex import VertexAIImageGenerator diff --git a/integrations/langfuse/CHANGELOG.md b/integrations/langfuse/CHANGELOG.md index 2efa17a68..0a90a7121 100644 --- a/integrations/langfuse/CHANGELOG.md +++ b/integrations/langfuse/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## [unreleased] + +### ๐Ÿšœ Refactor + +- Remove usage of deprecated `ChatMessage.to_openai_format` (#1001) + +### ๐Ÿ“š Documentation + +- Add link to langfuse in LangfuseConnector (#981) + +### ๐Ÿงช Testing + +- Do not retry tests in `hatch run test` command (#954) + +### โš™๏ธ Miscellaneous Tasks + +- Retry tests to reduce flakyness (#836) +- `Langfuse` - replace DynamicChatPromptBuilder with ChatPromptBuilder (#925) +- Remove all `DynamicChatPromptBuilder` references in Langfuse integration (#931) + ## [integrations/langfuse-v0.2.0] - 2024-06-18 ## [integrations/langfuse-v0.1.0] - 2024-06-13 diff --git a/integrations/mongodb_atlas/CHANGELOG.md b/integrations/mongodb_atlas/CHANGELOG.md index 851858355..91b073102 100644 --- a/integrations/mongodb_atlas/CHANGELOG.md +++ b/integrations/mongodb_atlas/CHANGELOG.md @@ -12,10 +12,16 @@ - Pass empty dict to filter instead of None (#775) - `Mongo` - Fallback to default filter policy when deserializing retrievers without the init parameter (#899) +### ๐Ÿงช Testing + +- Do not retry tests in `hatch run test` command (#954) + ### โš™๏ธ Miscellaneous Tasks - Retry tests to reduce flakyness (#836) - Update ruff invocation to include check parameter (#853) +- Update mongodb test for the new `apply_filter_policy` usage (#971) +- MongoDB - remove legacy filter support (#1066) ## [integrations/mongodb_atlas-v0.2.1] - 2024-04-09 diff --git a/integrations/mongodb_atlas/src/haystack_integrations/document_stores/mongodb_atlas/filters.py b/integrations/mongodb_atlas/src/haystack_integrations/document_stores/mongodb_atlas/filters.py index 4583d6cd3..0b5986222 100644 --- a/integrations/mongodb_atlas/src/haystack_integrations/document_stores/mongodb_atlas/filters.py +++ b/integrations/mongodb_atlas/src/haystack_integrations/document_stores/mongodb_atlas/filters.py @@ -5,7 +5,6 @@ from typing import Any, Dict from haystack.errors import FilterError -from haystack.utils.filters import convert from pandas import DataFrame UNSUPPORTED_TYPES_FOR_COMPARISON = (list, DataFrame) @@ -20,7 +19,8 @@ def _normalize_filters(filters: Dict[str, Any]) -> Dict[str, Any]: raise FilterError(msg) if "operator" not in filters and "conditions" not in filters: - filters = convert(filters) + msg = "Invalid filter syntax. See https://docs.haystack.deepset.ai/docs/metadata-filtering for details." + raise ValueError(msg) if "field" in filters: return _parse_comparison_condition(filters) diff --git a/integrations/ollama/CHANGELOG.md b/integrations/ollama/CHANGELOG.md index 6467aa868..8f51237e9 100644 --- a/integrations/ollama/CHANGELOG.md +++ b/integrations/ollama/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog +## [integrations/ollama-v1.0.0] - 2024-09-07 + +### ๐Ÿ› Bug Fixes + +- Chat roles for model responses in chat generators (#1030) + +### ๐Ÿšœ Refactor + +- [**breaking**] Use ollama python library instead of calling the API with `requests` (#1059) + +### ๐Ÿงช Testing + +- Do not retry tests in `hatch run test` command (#954) + +### โš™๏ธ Miscellaneous Tasks + +- Retry tests to reduce flakyness (#836) +- Update ruff invocation to include check parameter (#853) + ## [integrations/ollama-v0.0.7] - 2024-05-31 ### ๐Ÿš€ Features diff --git a/integrations/ollama/README.md b/integrations/ollama/README.md index c842cddf1..a8ec1d526 100644 --- a/integrations/ollama/README.md +++ b/integrations/ollama/README.md @@ -36,4 +36,4 @@ Then run tests: hatch run test ``` -The default model used here is ``orca-mini`` \ No newline at end of file +The default model used here is ``orca-mini`` for generation and ``nomic-embed-text`` for embeddings \ No newline at end of file diff --git a/integrations/ollama/examples/chat_generator_example.py b/integrations/ollama/examples/chat_generator_example.py index 2326ba708..3dfd01065 100644 --- a/integrations/ollama/examples/chat_generator_example.py +++ b/integrations/ollama/examples/chat_generator_example.py @@ -17,7 +17,7 @@ ), ChatMessage.from_user("How do I get started?"), ] -client = OllamaChatGenerator(model="orca-mini", timeout=45, url="http://localhost:11434/api/chat") +client = OllamaChatGenerator(model="orca-mini", timeout=45, url="http://localhost:11434") response = client.run(messages, generation_kwargs={"temperature": 0.2}) diff --git a/integrations/ollama/pyproject.toml b/integrations/ollama/pyproject.toml index 57aee153b..1174d3b78 100644 --- a/integrations/ollama/pyproject.toml +++ b/integrations/ollama/pyproject.toml @@ -27,7 +27,7 @@ classifiers = [ "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ] -dependencies = ["haystack-ai", "requests"] +dependencies = ["haystack-ai", "ollama"] [project.urls] Documentation = "https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/ollama#readme" @@ -161,5 +161,5 @@ markers = [ addopts = ["--import-mode=importlib"] [[tool.mypy.overrides]] -module = ["haystack.*", "haystack_integrations.*", "pytest.*"] +module = ["haystack.*", "haystack_integrations.*", "pytest.*", "ollama.*"] ignore_missing_imports = true diff --git a/integrations/ollama/src/haystack_integrations/components/embedders/ollama/document_embedder.py b/integrations/ollama/src/haystack_integrations/components/embedders/ollama/document_embedder.py index b5783c611..ac8f38f35 100644 --- a/integrations/ollama/src/haystack_integrations/components/embedders/ollama/document_embedder.py +++ b/integrations/ollama/src/haystack_integrations/components/embedders/ollama/document_embedder.py @@ -1,9 +1,10 @@ from typing import Any, Dict, List, Optional -import requests from haystack import Document, component from tqdm import tqdm +from ollama import Client + @component class OllamaDocumentEmbedder: @@ -27,7 +28,7 @@ class OllamaDocumentEmbedder: def __init__( self, model: str = "nomic-embed-text", - url: str = "http://localhost:11434/api/embeddings", + url: str = "http://localhost:11434", generation_kwargs: Optional[Dict[str, Any]] = None, timeout: int = 120, prefix: str = "", @@ -40,7 +41,7 @@ def __init__( :param model: The name of the model to use. The model should be available in the running Ollama instance. :param url: - The URL of the chat endpoint of a running Ollama instance. + The URL of a running Ollama instance. :param generation_kwargs: Optional arguments to pass to the Ollama generation endpoint, such as temperature, top_p, and others. See the available arguments in @@ -59,11 +60,7 @@ def __init__( self.suffix = suffix self.prefix = prefix - def _create_json_payload(self, text: str, generation_kwargs: Optional[Dict[str, Any]]) -> Dict[str, Any]: - """ - Returns A dictionary of JSON arguments for a POST request to an Ollama service - """ - return {"model": self.model, "prompt": text, "options": {**self.generation_kwargs, **(generation_kwargs or {})}} + self._client = Client(host=self.url, timeout=self.timeout) def _prepare_texts_to_embed(self, documents: List[Document]) -> List[str]: """ @@ -103,10 +100,7 @@ def _embed_batch( range(0, len(texts_to_embed), batch_size), disable=not self.progress_bar, desc="Calculating embeddings" ): batch = texts_to_embed[i] # Single batch only - payload = self._create_json_payload(batch, generation_kwargs) - response = requests.post(url=self.url, json=payload, timeout=self.timeout) - response.raise_for_status() - result = response.json() + result = self._client.embeddings(model=self.model, prompt=batch, options=generation_kwargs) all_embeddings.append(result["embedding"]) meta["model"] = self.model diff --git a/integrations/ollama/src/haystack_integrations/components/embedders/ollama/text_embedder.py b/integrations/ollama/src/haystack_integrations/components/embedders/ollama/text_embedder.py index 5a28ba393..7779c6d6e 100644 --- a/integrations/ollama/src/haystack_integrations/components/embedders/ollama/text_embedder.py +++ b/integrations/ollama/src/haystack_integrations/components/embedders/ollama/text_embedder.py @@ -1,8 +1,9 @@ from typing import Any, Dict, List, Optional -import requests from haystack import component +from ollama import Client + @component class OllamaTextEmbedder: @@ -23,7 +24,7 @@ class OllamaTextEmbedder: def __init__( self, model: str = "nomic-embed-text", - url: str = "http://localhost:11434/api/embeddings", + url: str = "http://localhost:11434", generation_kwargs: Optional[Dict[str, Any]] = None, timeout: int = 120, ): @@ -31,7 +32,7 @@ def __init__( :param model: The name of the model to use. The model should be available in the running Ollama instance. :param url: - The URL of the chat endpoint of a running Ollama instance. + The URL of a running Ollama instance. :param generation_kwargs: Optional arguments to pass to the Ollama generation endpoint, such as temperature, top_p, and others. See the available arguments in @@ -44,11 +45,7 @@ def __init__( self.url = url self.model = model - def _create_json_payload(self, text: str, generation_kwargs: Optional[Dict[str, Any]]) -> Dict[str, Any]: - """ - Returns A dictionary of JSON arguments for a POST request to an Ollama service - """ - return {"model": self.model, "prompt": text, "options": {**self.generation_kwargs, **(generation_kwargs or {})}} + self._client = Client(host=self.url, timeout=self.timeout) @component.output_types(embedding=List[float], meta=Dict[str, Any]) def run(self, text: str, generation_kwargs: Optional[Dict[str, Any]] = None): @@ -65,14 +62,7 @@ def run(self, text: str, generation_kwargs: Optional[Dict[str, Any]] = None): - `embedding`: The computed embeddings - `meta`: The metadata collected during the embedding process """ - - payload = self._create_json_payload(text, generation_kwargs) - - response = requests.post(url=self.url, json=payload, timeout=self.timeout) - - response.raise_for_status() - - result = response.json() - result["meta"] = {"model": self.model, "duration": response.elapsed} + result = self._client.embeddings(model=self.model, prompt=text, options=generation_kwargs) + result["meta"] = {"model": self.model} return result diff --git a/integrations/ollama/src/haystack_integrations/components/generators/ollama/chat/chat_generator.py b/integrations/ollama/src/haystack_integrations/components/generators/ollama/chat/chat_generator.py index a95d8c4fb..1f3a0bf1e 100644 --- a/integrations/ollama/src/haystack_integrations/components/generators/ollama/chat/chat_generator.py +++ b/integrations/ollama/src/haystack_integrations/components/generators/ollama/chat/chat_generator.py @@ -1,10 +1,9 @@ -import json from typing import Any, Callable, Dict, List, Optional -import requests from haystack import component from haystack.dataclasses import ChatMessage, StreamingChunk -from requests import Response + +from ollama import Client @component @@ -19,7 +18,7 @@ class OllamaChatGenerator: from haystack.dataclasses import ChatMessage generator = OllamaChatGenerator(model="zephyr", - url = "http://localhost:11434/api/chat", + url = "http://localhost:11434", generation_kwargs={ "num_predict": 100, "temperature": 0.9, @@ -35,9 +34,8 @@ class OllamaChatGenerator: def __init__( self, model: str = "orca-mini", - url: str = "http://localhost:11434/api/chat", + url: str = "http://localhost:11434", generation_kwargs: Optional[Dict[str, Any]] = None, - template: Optional[str] = None, timeout: int = 120, streaming_callback: Optional[Callable[[StreamingChunk], None]] = None, ): @@ -45,13 +43,11 @@ def __init__( :param model: The name of the model to use. The model should be available in the running Ollama instance. :param url: - The URL of the chat endpoint of a running Ollama instance. + The URL of a running Ollama instance. :param generation_kwargs: Optional arguments to pass to the Ollama generation endpoint, such as temperature, top_p, and others. See the available arguments in [Ollama docs](https://github.com/jmorganca/ollama/blob/main/docs/modelfile.md#valid-parameters-and-values). - :param template: - The full prompt template (overrides what is defined in the Ollama Modelfile). :param timeout: The number of seconds before throwing a timeout error from the Ollama API. :param streaming_callback: @@ -60,35 +56,22 @@ def __init__( """ self.timeout = timeout - self.template = template self.generation_kwargs = generation_kwargs or {} self.url = url self.model = model self.streaming_callback = streaming_callback + self._client = Client(host=self.url, timeout=self.timeout) + def _message_to_dict(self, message: ChatMessage) -> Dict[str, str]: return {"role": message.role.value, "content": message.content} - def _create_json_payload(self, messages: List[ChatMessage], stream=False, generation_kwargs=None) -> Dict[str, Any]: - """ - Returns A dictionary of JSON arguments for a POST request to an Ollama service - """ - generation_kwargs = generation_kwargs or {} - return { - "messages": [self._message_to_dict(message) for message in messages], - "model": self.model, - "stream": stream, - "template": self.template, - "options": generation_kwargs, - } - - def _build_message_from_ollama_response(self, ollama_response: Response) -> ChatMessage: + def _build_message_from_ollama_response(self, ollama_response: Dict[str, Any]) -> ChatMessage: """ Converts the non-streaming response from the Ollama API to a ChatMessage. """ - json_content = ollama_response.json() - message = ChatMessage.from_assistant(content=json_content["message"]["content"]) - message.meta.update({key: value for key, value in json_content.items() if key != "message"}) + message = ChatMessage.from_assistant(content=ollama_response["message"]["content"]) + message.meta.update({key: value for key, value in ollama_response.items() if key != "message"}) return message def _convert_to_streaming_response(self, chunks: List[StreamingChunk]) -> Dict[str, List[Any]]: @@ -105,11 +88,9 @@ def _build_chunk(self, chunk_response: Any) -> StreamingChunk: """ Converts the response from the Ollama API to a StreamingChunk. """ - decoded_chunk = json.loads(chunk_response.decode("utf-8")) - - content = decoded_chunk["message"]["content"] - meta = {key: value for key, value in decoded_chunk.items() if key != "message"} - meta["role"] = decoded_chunk["message"]["role"] + content = chunk_response["message"]["content"] + meta = {key: value for key, value in chunk_response.items() if key != "message"} + meta["role"] = chunk_response["message"]["role"] chunk_message = StreamingChunk(content, meta) return chunk_message @@ -119,7 +100,7 @@ def _handle_streaming_response(self, response) -> List[StreamingChunk]: Handles Streaming response cases """ chunks: List[StreamingChunk] = [] - for chunk in response.iter_lines(): + for chunk in response: chunk_delta: StreamingChunk = self._build_chunk(chunk) chunks.append(chunk_delta) if self.streaming_callback is not None: @@ -149,13 +130,8 @@ def run( generation_kwargs = {**self.generation_kwargs, **(generation_kwargs or {})} stream = self.streaming_callback is not None - - json_payload = self._create_json_payload(messages, stream, generation_kwargs) - - response = requests.post(url=self.url, json=json_payload, timeout=self.timeout, stream=stream) - - # throw error on unsuccessful response - response.raise_for_status() + messages = [self._message_to_dict(message) for message in messages] + response = self._client.chat(model=self.model, messages=messages, stream=stream, options=generation_kwargs) if stream: chunks: List[StreamingChunk] = self._handle_streaming_response(response) diff --git a/integrations/ollama/src/haystack_integrations/components/generators/ollama/generator.py b/integrations/ollama/src/haystack_integrations/components/generators/ollama/generator.py index 50c65b650..d92932c3e 100644 --- a/integrations/ollama/src/haystack_integrations/components/generators/ollama/generator.py +++ b/integrations/ollama/src/haystack_integrations/components/generators/ollama/generator.py @@ -1,11 +1,10 @@ -import json from typing import Any, Callable, Dict, List, Optional -import requests from haystack import component, default_from_dict, default_to_dict from haystack.dataclasses import StreamingChunk from haystack.utils.callable_serialization import deserialize_callable, serialize_callable -from requests import Response + +from ollama import Client @component @@ -18,7 +17,7 @@ class OllamaGenerator: from haystack_integrations.components.generators.ollama import OllamaGenerator generator = OllamaGenerator(model="zephyr", - url = "http://localhost:11434/api/generate", + url = "http://localhost:11434", generation_kwargs={ "num_predict": 100, "temperature": 0.9, @@ -31,7 +30,7 @@ class OllamaGenerator: def __init__( self, model: str = "orca-mini", - url: str = "http://localhost:11434/api/generate", + url: str = "http://localhost:11434", generation_kwargs: Optional[Dict[str, Any]] = None, system_prompt: Optional[str] = None, template: Optional[str] = None, @@ -43,7 +42,7 @@ def __init__( :param model: The name of the model to use. The model should be available in the running Ollama instance. :param url: - The URL of the generation endpoint of a running Ollama instance. + The URL of a running Ollama instance. :param generation_kwargs: Optional arguments to pass to the Ollama generation endpoint, such as temperature, top_p, and others. See the available arguments in @@ -70,6 +69,8 @@ def __init__( self.generation_kwargs = generation_kwargs or {} self.streaming_callback = streaming_callback + self._client = Client(host=self.url, timeout=self.timeout) + def to_dict(self) -> Dict[str, Any]: """ Serializes the component to a dictionary. @@ -106,30 +107,13 @@ def from_dict(cls, data: Dict[str, Any]) -> "OllamaGenerator": data["init_parameters"]["streaming_callback"] = deserialize_callable(serialized_callback_handler) return default_from_dict(cls, data) - def _create_json_payload(self, prompt: str, stream: bool, generation_kwargs=None) -> Dict[str, Any]: - """ - Returns a dictionary of JSON arguments for a POST request to an Ollama service. - """ - generation_kwargs = generation_kwargs or {} - return { - "prompt": prompt, - "model": self.model, - "stream": stream, - "raw": self.raw, - "template": self.template, - "system": self.system_prompt, - "options": generation_kwargs, - } - - def _convert_to_response(self, ollama_response: Response) -> Dict[str, List[Any]]: + def _convert_to_response(self, ollama_response: Dict[str, Any]) -> Dict[str, List[Any]]: """ Converts a response from the Ollama API to the required Haystack format. """ - resp_dict = ollama_response.json() - - replies = [resp_dict["response"]] - meta = {key: value for key, value in resp_dict.items() if key != "response"} + replies = [ollama_response["response"]] + meta = {key: value for key, value in ollama_response.items() if key != "response"} return {"replies": replies, "meta": [meta]} @@ -148,7 +132,7 @@ def _handle_streaming_response(self, response) -> List[StreamingChunk]: Handles Streaming response cases """ chunks: List[StreamingChunk] = [] - for chunk in response.iter_lines(): + for chunk in response: chunk_delta: StreamingChunk = self._build_chunk(chunk) chunks.append(chunk_delta) if self.streaming_callback is not None: @@ -159,10 +143,8 @@ def _build_chunk(self, chunk_response: Any) -> StreamingChunk: """ Converts the response from the Ollama API to a StreamingChunk. """ - decoded_chunk = json.loads(chunk_response.decode("utf-8")) - - content = decoded_chunk["response"] - meta = {key: value for key, value in decoded_chunk.items() if key != "response"} + content = chunk_response["response"] + meta = {key: value for key, value in chunk_response.items() if key != "response"} chunk_message = StreamingChunk(content, meta) return chunk_message @@ -190,12 +172,7 @@ def run( stream = self.streaming_callback is not None - json_payload = self._create_json_payload(prompt, stream, generation_kwargs) - - response = requests.post(url=self.url, json=json_payload, timeout=self.timeout, stream=stream) - - # throw error on unsuccessful response - response.raise_for_status() + response = self._client.generate(model=self.model, prompt=prompt, stream=stream, options=generation_kwargs) if stream: chunks: List[StreamingChunk] = self._handle_streaming_response(response) diff --git a/integrations/ollama/tests/test_chat_generator.py b/integrations/ollama/tests/test_chat_generator.py index dd4e746aa..79d70675a 100644 --- a/integrations/ollama/tests/test_chat_generator.py +++ b/integrations/ollama/tests/test_chat_generator.py @@ -3,7 +3,7 @@ import pytest from haystack.dataclasses import ChatMessage, ChatRole -from requests import HTTPError, Response +from ollama._types import ResponseError from haystack_integrations.components.generators.ollama import OllamaChatGenerator @@ -22,47 +22,27 @@ class TestOllamaChatGenerator: def test_init_default(self): component = OllamaChatGenerator() assert component.model == "orca-mini" - assert component.url == "http://localhost:11434/api/chat" + assert component.url == "http://localhost:11434" assert component.generation_kwargs == {} - assert component.template is None assert component.timeout == 120 def test_init(self): component = OllamaChatGenerator( model="llama2", - url="http://my-custom-endpoint:11434/api/chat", + url="http://my-custom-endpoint:11434", generation_kwargs={"temperature": 0.5}, timeout=5, ) assert component.model == "llama2" - assert component.url == "http://my-custom-endpoint:11434/api/chat" + assert component.url == "http://my-custom-endpoint:11434" assert component.generation_kwargs == {"temperature": 0.5} - assert component.template is None assert component.timeout == 5 - def test_create_json_payload(self, chat_messages): - observed = OllamaChatGenerator(model="some_model")._create_json_payload( - chat_messages, False, {"temperature": 0.1} - ) - expected = { - "messages": [ - {"role": "user", "content": "Tell me about why Super Mario is the greatest superhero"}, - {"role": "assistant", "content": "Super Mario has prevented Bowser from destroying the world"}, - ], - "model": "some_model", - "stream": False, - "template": None, - "options": {"temperature": 0.1}, - } - - assert observed == expected - def test_build_message_from_ollama_response(self): model = "some_model" - mock_ollama_response = Mock(Response) - mock_ollama_response.json.return_value = { + ollama_response = { "model": model, "created_at": "2023-12-12T14:13:43.416799Z", "message": {"role": "assistant", "content": "Hello! How are you today?"}, @@ -75,7 +55,7 @@ def test_build_message_from_ollama_response(self): "eval_duration": 4799921000, } - observed = OllamaChatGenerator(model=model)._build_message_from_ollama_response(mock_ollama_response) + observed = OllamaChatGenerator(model=model)._build_message_from_ollama_response(ollama_response) assert observed.role == "assistant" assert observed.content == "Hello! How are you today?" @@ -123,7 +103,7 @@ def test_run_with_chat_history(self): def test_run_model_unavailable(self): component = OllamaChatGenerator(model="Alistair_and_Stefano_are_great") - with pytest.raises(HTTPError): + with pytest.raises(ResponseError): message = ChatMessage.from_user( "Based on your infinite wisdom, can you tell me why Alistair and Stefano are so great?" ) diff --git a/integrations/ollama/tests/test_document_embedder.py b/integrations/ollama/tests/test_document_embedder.py index 0f5b55881..4fe3cfbb3 100644 --- a/integrations/ollama/tests/test_document_embedder.py +++ b/integrations/ollama/tests/test_document_embedder.py @@ -1,6 +1,6 @@ import pytest from haystack import Document -from requests import HTTPError +from ollama._types import ResponseError from haystack_integrations.components.embedders.ollama import OllamaDocumentEmbedder @@ -11,27 +11,27 @@ def test_init_defaults(self): assert embedder.timeout == 120 assert embedder.generation_kwargs == {} - assert embedder.url == "http://localhost:11434/api/embeddings" + assert embedder.url == "http://localhost:11434" assert embedder.model == "nomic-embed-text" def test_init(self): embedder = OllamaDocumentEmbedder( model="nomic-embed-text", - url="http://my-custom-endpoint:11434/api/embeddings", + url="http://my-custom-endpoint:11434", generation_kwargs={"temperature": 0.5}, timeout=3000, ) assert embedder.timeout == 3000 assert embedder.generation_kwargs == {"temperature": 0.5} - assert embedder.url == "http://my-custom-endpoint:11434/api/embeddings" + assert embedder.url == "http://my-custom-endpoint:11434" assert embedder.model == "nomic-embed-text" @pytest.mark.integration def test_model_not_found(self): embedder = OllamaDocumentEmbedder(model="cheese") - with pytest.raises(HTTPError): + with pytest.raises(ResponseError): embedder.run([Document("hello")]) @pytest.mark.integration diff --git a/integrations/ollama/tests/test_generator.py b/integrations/ollama/tests/test_generator.py index 069bbd227..c4c6906db 100644 --- a/integrations/ollama/tests/test_generator.py +++ b/integrations/ollama/tests/test_generator.py @@ -2,12 +2,10 @@ # # SPDX-License-Identifier: Apache-2.0 -from typing import Any - import pytest from haystack.components.generators.utils import print_streaming_chunk from haystack.dataclasses import StreamingChunk -from requests import HTTPError +from ollama._types import ResponseError from haystack_integrations.components.generators.ollama import OllamaGenerator @@ -35,13 +33,13 @@ def test_run_capital_cities(self): def test_run_model_unavailable(self): component = OllamaGenerator(model="Alistair_is_great") - with pytest.raises(HTTPError): + with pytest.raises(ResponseError): component.run(prompt="Why is Alistair so great?") def test_init_default(self): component = OllamaGenerator() assert component.model == "orca-mini" - assert component.url == "http://localhost:11434/api/generate" + assert component.url == "http://localhost:11434" assert component.generation_kwargs == {} assert component.system_prompt is None assert component.template is None @@ -55,14 +53,14 @@ def callback(x: StreamingChunk): component = OllamaGenerator( model="llama2", - url="http://my-custom-endpoint:11434/api/generate", + url="http://my-custom-endpoint:11434", generation_kwargs={"temperature": 0.5}, system_prompt="You are Luigi from Super Mario Bros.", timeout=5, streaming_callback=callback, ) assert component.model == "llama2" - assert component.url == "http://my-custom-endpoint:11434/api/generate" + assert component.url == "http://my-custom-endpoint:11434" assert component.generation_kwargs == {"temperature": 0.5} assert component.system_prompt == "You are Luigi from Super Mario Bros." assert component.template is None @@ -80,7 +78,7 @@ def callback(x: StreamingChunk): "template": None, "system_prompt": None, "model": "orca-mini", - "url": "http://localhost:11434/api/generate", + "url": "http://localhost:11434", "streaming_callback": None, "generation_kwargs": {}, }, @@ -128,44 +126,6 @@ def test_from_dict(self): assert component.url == "going_to_51_pegasi_b_for_weekend" assert component.generation_kwargs == {"max_tokens": 10, "some_test_param": "test-params"} - @pytest.mark.parametrize( - "configuration", - [ - { - "model": "some_model", - "url": "https://localhost:11434/api/generate", - "raw": True, - "system_prompt": "You are mario from Super Mario Bros.", - "template": None, - }, - { - "model": "some_model2", - "url": "https://localhost:11434/api/generate", - "raw": False, - "system_prompt": None, - "template": "some template", - }, - ], - ) - @pytest.mark.parametrize("stream", [True, False]) - def test_create_json_payload(self, configuration: dict[str, Any], stream: bool): - prompt = "hello" - component = OllamaGenerator(**configuration) - - observed = component._create_json_payload(prompt=prompt, stream=stream) - - expected = { - "prompt": prompt, - "model": configuration["model"], - "stream": stream, - "system": configuration["system_prompt"], - "raw": configuration["raw"], - "template": configuration["template"], - "options": {}, - } - - assert observed == expected - @pytest.mark.integration def test_ollama_generator_run_streaming(self): class Callback: diff --git a/integrations/ollama/tests/test_text_embedder.py b/integrations/ollama/tests/test_text_embedder.py index e7b69460f..d0f74c377 100644 --- a/integrations/ollama/tests/test_text_embedder.py +++ b/integrations/ollama/tests/test_text_embedder.py @@ -1,5 +1,5 @@ import pytest -from requests import HTTPError +from ollama._types import ResponseError from haystack_integrations.components.embedders.ollama import OllamaTextEmbedder @@ -10,27 +10,27 @@ def test_init_defaults(self): assert embedder.timeout == 120 assert embedder.generation_kwargs == {} - assert embedder.url == "http://localhost:11434/api/embeddings" + assert embedder.url == "http://localhost:11434" assert embedder.model == "nomic-embed-text" def test_init(self): embedder = OllamaTextEmbedder( model="llama2", - url="http://my-custom-endpoint:11434/api/embeddings", + url="http://my-custom-endpoint:11434", generation_kwargs={"temperature": 0.5}, timeout=3000, ) assert embedder.timeout == 3000 assert embedder.generation_kwargs == {"temperature": 0.5} - assert embedder.url == "http://my-custom-endpoint:11434/api/embeddings" + assert embedder.url == "http://my-custom-endpoint:11434" assert embedder.model == "llama2" @pytest.mark.integration def test_model_not_found(self): embedder = OllamaTextEmbedder(model="cheese") - with pytest.raises(HTTPError): + with pytest.raises(ResponseError): embedder.run("hello") @pytest.mark.integration diff --git a/integrations/opensearch/CHANGELOG.md b/integrations/opensearch/CHANGELOG.md index 6509d1e0f..713848915 100644 --- a/integrations/opensearch/CHANGELOG.md +++ b/integrations/opensearch/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## [integrations/opensearch-v1.0.0] - 2024-09-12 + +### ๐Ÿ“š Documentation + +- Update opensearch retriever docstrings (#1035) + +### ๐Ÿงช Testing + +- Do not retry tests in `hatch run test` command (#954) + +### โš™๏ธ Miscellaneous Tasks + +- OpenSearch - remove legacy filter support (#1067) + +### Docs + +- Update BM25 docstrings (#945) + ## [integrations/opensearch-v0.9.0] - 2024-08-01 ### ๐Ÿš€ Features diff --git a/integrations/opensearch/src/haystack_integrations/document_stores/opensearch/document_store.py b/integrations/opensearch/src/haystack_integrations/document_stores/opensearch/document_store.py index 3a6056bd2..6f7a6c96e 100644 --- a/integrations/opensearch/src/haystack_integrations/document_stores/opensearch/document_store.py +++ b/integrations/opensearch/src/haystack_integrations/document_stores/opensearch/document_store.py @@ -9,7 +9,6 @@ from haystack.dataclasses import Document from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError from haystack.document_stores.types import DuplicatePolicy -from haystack.utils.filters import convert from opensearchpy import OpenSearch from opensearchpy.helpers import bulk @@ -238,14 +237,14 @@ def _search_documents(self, **kwargs) -> List[Document]: def filter_documents(self, filters: Optional[Dict[str, Any]] = None) -> List[Document]: if filters and "operator" not in filters and "conditions" not in filters: - filters = convert(filters) + msg = "Invalid filter syntax. See https://docs.haystack.deepset.ai/docs/metadata-filtering for details." + raise ValueError(msg) if filters: query = {"bool": {"filter": normalize_filters(filters)}} documents = self._search_documents(query=query, size=10_000) else: documents = self._search_documents(size=10_000) - return documents def write_documents(self, documents: List[Document], policy: DuplicatePolicy = DuplicatePolicy.NONE) -> int: @@ -384,7 +383,8 @@ def _bm25_retrieval( :returns: List of Document that match `query` """ if filters and "operator" not in filters and "conditions" not in filters: - filters = convert(filters) + msg = "Invalid filter syntax. See https://docs.haystack.deepset.ai/docs/metadata-filtering for details." + raise ValueError(msg) if not query: body: Dict[str, Any] = {"query": {"bool": {"must": {"match_all": {}}}}} @@ -478,7 +478,8 @@ def _embedding_retrieval( :returns: List of Document that are most similar to `query_embedding` """ if filters and "operator" not in filters and "conditions" not in filters: - filters = convert(filters) + msg = "Legacy filters support has been removed. Please see documentation for new filter syntax." + raise ValueError(msg) if not query_embedding: msg = "query_embedding must be a non-empty list of floats" diff --git a/integrations/opensearch/tests/test_document_store.py b/integrations/opensearch/tests/test_document_store.py index a7d516b3e..9cc4bf4ea 100644 --- a/integrations/opensearch/tests/test_document_store.py +++ b/integrations/opensearch/tests/test_document_store.py @@ -574,76 +574,6 @@ def test_bm25_retrieval_with_filters(self, document_store: OpenSearchDocumentSto retrieved_ids = sorted([doc.id for doc in res]) assert retrieved_ids == ["1", "2", "3", "4", "5"] - def test_bm25_retrieval_with_legacy_filters(self, document_store: OpenSearchDocumentStore): - document_store.write_documents( - [ - Document( - content="Haskell is a functional programming language", - meta={"likes": 100000, "language_type": "functional"}, - id="1", - ), - Document( - content="Lisp is a functional programming language", - meta={"likes": 10000, "language_type": "functional"}, - id="2", - ), - Document( - content="Exilir is a functional programming language", - meta={"likes": 1000, "language_type": "functional"}, - id="3", - ), - Document( - content="F# is a functional programming language", - meta={"likes": 100, "language_type": "functional"}, - id="4", - ), - Document( - content="C# is a functional programming language", - meta={"likes": 10, "language_type": "functional"}, - id="5", - ), - Document( - content="C++ is an object oriented programming language", - meta={"likes": 100000, "language_type": "object_oriented"}, - id="6", - ), - Document( - content="Dart is an object oriented programming language", - meta={"likes": 10000, "language_type": "object_oriented"}, - id="7", - ), - Document( - content="Go is an object oriented programming language", - meta={"likes": 1000, "language_type": "object_oriented"}, - id="8", - ), - Document( - content="Python is a object oriented programming language", - meta={"likes": 100, "language_type": "object_oriented"}, - id="9", - ), - Document( - content="Ruby is a object oriented programming language", - meta={"likes": 10, "language_type": "object_oriented"}, - id="10", - ), - Document( - content="PHP is a object oriented programming language", - meta={"likes": 1, "language_type": "object_oriented"}, - id="11", - ), - ] - ) - - res = document_store._bm25_retrieval( - "programming", - top_k=10, - filters={"language_type": "functional"}, - ) - assert len(res) == 5 - retrieved_ids = sorted([doc.id for doc in res]) - assert retrieved_ids == ["1", "2", "3", "4", "5"] - def test_bm25_retrieval_with_custom_query(self, document_store: OpenSearchDocumentStore): document_store.write_documents( [ @@ -760,27 +690,6 @@ def test_embedding_retrieval_with_filters(self, document_store_embedding_dim_4: assert len(results) == 1 assert results[0].content == "Not very similar document with meta field" - def test_embedding_retrieval_with_legacy_filters(self, document_store_embedding_dim_4: OpenSearchDocumentStore): - docs = [ - Document(content="Most similar document", embedding=[1.0, 1.0, 1.0, 1.0]), - Document(content="2nd best document", embedding=[0.8, 0.8, 0.8, 1.0]), - Document( - content="Not very similar document with meta field", - embedding=[0.0, 0.8, 0.3, 0.9], - meta={"meta_field": "custom_value"}, - ), - ] - document_store_embedding_dim_4.write_documents(docs) - - filters = {"meta_field": "custom_value"} - # we set top_k=3, to make the test pass as we are not sure whether efficient filtering is supported for nmslib - # TODO: remove top_k=3, when efficient filtering is supported for nmslib - results = document_store_embedding_dim_4._embedding_retrieval( - query_embedding=[0.1, 0.1, 0.1, 0.1], top_k=3, filters=filters - ) - assert len(results) == 1 - assert results[0].content == "Not very similar document with meta field" - def test_embedding_retrieval_pagination(self, document_store_embedding_dim_4: OpenSearchDocumentStore): """ Test that handling of pagination works as expected, when the matching documents are > 10. diff --git a/integrations/pgvector/CHANGELOG.md b/integrations/pgvector/CHANGELOG.md index deb6faece..0fe5f4fa4 100644 --- a/integrations/pgvector/CHANGELOG.md +++ b/integrations/pgvector/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [unreleased] +## [integrations/pgvector-v1.0.0] - 2024-09-12 ### ๐Ÿš€ Features @@ -10,10 +10,15 @@ - `PgVector` - Fallback to default filter policy when deserializing retrievers without the init parameter (#900) +### ๐Ÿงช Testing + +- Do not retry tests in `hatch run test` command (#954) + ### โš™๏ธ Miscellaneous Tasks - Retry tests to reduce flakyness (#836) - Update ruff invocation to include check parameter (#853) +- PgVector - remove legacy filter support (#1068) ## [integrations/pgvector-v0.4.0] - 2024-06-20 diff --git a/integrations/pgvector/src/haystack_integrations/document_stores/pgvector/document_store.py b/integrations/pgvector/src/haystack_integrations/document_stores/pgvector/document_store.py index ae4878aba..a02c46200 100644 --- a/integrations/pgvector/src/haystack_integrations/document_stores/pgvector/document_store.py +++ b/integrations/pgvector/src/haystack_integrations/document_stores/pgvector/document_store.py @@ -9,7 +9,6 @@ from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError from haystack.document_stores.types import DuplicatePolicy from haystack.utils.auth import Secret, deserialize_secrets_inplace -from haystack.utils.filters import convert from psycopg import Error, IntegrityError, connect from psycopg.abc import Query from psycopg.cursor import Cursor @@ -389,7 +388,8 @@ def filter_documents(self, filters: Optional[Dict[str, Any]] = None) -> List[Doc msg = "Filters must be a dictionary" raise TypeError(msg) if "operator" not in filters and "conditions" not in filters: - filters = convert(filters) + msg = "Invalid filter syntax. See https://docs.haystack.deepset.ai/docs/metadata-filtering for details." + raise ValueError(msg) sql_filter = SQL("SELECT * FROM {table_name}").format(table_name=Identifier(self.table_name)) diff --git a/integrations/pinecone/CHANGELOG.md b/integrations/pinecone/CHANGELOG.md index a041d63de..7810e486c 100644 --- a/integrations/pinecone/CHANGELOG.md +++ b/integrations/pinecone/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## [integrations/pinecone-v2.0.0] - 2024-09-12 + +### โš™๏ธ Miscellaneous Tasks + +- Pinecone - remove legacy filter support (#1069) + ## [integrations/pinecone-v1.2.3] - 2024-08-29 ### ๐Ÿš€ Features diff --git a/integrations/pinecone/src/haystack_integrations/document_stores/pinecone/document_store.py b/integrations/pinecone/src/haystack_integrations/document_stores/pinecone/document_store.py index 75d6270ca..07f217f5b 100644 --- a/integrations/pinecone/src/haystack_integrations/document_stores/pinecone/document_store.py +++ b/integrations/pinecone/src/haystack_integrations/document_stores/pinecone/document_store.py @@ -11,7 +11,6 @@ from haystack.dataclasses import Document from haystack.document_stores.types import DuplicatePolicy from haystack.utils import Secret, deserialize_secrets_inplace -from haystack.utils.filters import convert from pinecone import Pinecone, PodSpec, ServerlessSpec @@ -201,6 +200,10 @@ def filter_documents(self, filters: Optional[Dict[str, Any]] = None) -> List[Doc :returns: A list of Documents that match the given filters. """ + if filters and "operator" not in filters and "conditions" not in filters: + msg = "Invalid filter syntax. See https://docs.haystack.deepset.ai/docs/metadata-filtering for details." + raise ValueError(msg) + # Pinecone only performs vector similarity search # here we are querying with a dummy vector and the max compatible top_k documents = self._embedding_retrieval(query_embedding=self._dummy_vector, filters=filters, top_k=TOP_K_LIMIT) @@ -253,7 +256,8 @@ def _embedding_retrieval( raise ValueError(msg) if filters and "operator" not in filters and "conditions" not in filters: - filters = convert(filters) + msg = "Legacy filters support has been removed. Please see documentation for new filter syntax." + raise ValueError(msg) filters = _normalize_filters(filters) if filters else None result = self.index.query( diff --git a/integrations/qdrant/CHANGELOG.md b/integrations/qdrant/CHANGELOG.md index ad664bdd4..edc936fb2 100644 --- a/integrations/qdrant/CHANGELOG.md +++ b/integrations/qdrant/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## [integrations/qdrant-v5.1.0] - 2024-09-12 + +### ๐Ÿš€ Features + +- Qdrant - Add group_by and group_size optional parameters to Retrievers (#1054) + ## [integrations/qdrant-v5.0.0] - 2024-09-02 ## [integrations/qdrant-v4.2.0] - 2024-08-27 diff --git a/integrations/qdrant/src/haystack_integrations/components/retrievers/qdrant/retriever.py b/integrations/qdrant/src/haystack_integrations/components/retrievers/qdrant/retriever.py index 408b2458a..fee9a6182 100644 --- a/integrations/qdrant/src/haystack_integrations/components/retrievers/qdrant/retriever.py +++ b/integrations/qdrant/src/haystack_integrations/components/retrievers/qdrant/retriever.py @@ -44,13 +44,16 @@ def __init__( return_embedding: bool = False, filter_policy: Union[str, FilterPolicy] = FilterPolicy.REPLACE, score_threshold: Optional[float] = None, + group_by: Optional[str] = None, + group_size: Optional[int] = None, ): """ Create a QdrantEmbeddingRetriever component. :param document_store: An instance of QdrantDocumentStore. :param filters: A dictionary with filters to narrow down the search space. - :param top_k: The maximum number of documents to retrieve. + :param top_k: The maximum number of documents to retrieve. If using `group_by` parameters, maximum number of + groups to return. :param scale_score: Whether to scale the scores of the retrieved documents or not. :param return_embedding: Whether to return the embedding of the retrieved Documents. :param filter_policy: Policy to determine how filters are applied. @@ -58,6 +61,9 @@ def __init__( Score of the returned result might be higher or smaller than the threshold depending on the `similarity` function specified in the Document Store. E.g. for cosine similarity only higher scores will be returned. + :param group_by: Payload field to group by, must be a string or number field. If the field contains more than 1 + value, all values will be used for grouping. One point can be in multiple groups. + :param group_size: Maximum amount of points to return per group. Default is 3. :raises ValueError: If `document_store` is not an instance of `QdrantDocumentStore`. """ @@ -75,6 +81,8 @@ def __init__( filter_policy if isinstance(filter_policy, FilterPolicy) else FilterPolicy.from_str(filter_policy) ) self._score_threshold = score_threshold + self._group_by = group_by + self._group_size = group_size def to_dict(self) -> Dict[str, Any]: """ @@ -92,6 +100,8 @@ def to_dict(self) -> Dict[str, Any]: scale_score=self._scale_score, return_embedding=self._return_embedding, score_threshold=self._score_threshold, + group_by=self._group_by, + group_size=self._group_size, ) d["init_parameters"]["document_store"] = self._document_store.to_dict() @@ -124,16 +134,22 @@ def run( scale_score: Optional[bool] = None, return_embedding: Optional[bool] = None, score_threshold: Optional[float] = None, + group_by: Optional[str] = None, + group_size: Optional[int] = None, ): """ Run the Embedding Retriever on the given input data. :param query_embedding: Embedding of the query. :param filters: A dictionary with filters to narrow down the search space. - :param top_k: The maximum number of documents to return. + :param top_k: The maximum number of documents to return. If using `group_by` parameters, maximum number of + groups to return. :param scale_score: Whether to scale the scores of the retrieved documents or not. :param return_embedding: Whether to return the embedding of the retrieved Documents. :param score_threshold: A minimal score threshold for the result. + :param group_by: Payload field to group by, must be a string or number field. If the field contains more than 1 + value, all values will be used for grouping. One point can be in multiple groups. + :param group_size: Maximum amount of points to return per group. Default is 3. :returns: The retrieved documents. @@ -147,6 +163,8 @@ def run( scale_score=scale_score or self._scale_score, return_embedding=return_embedding or self._return_embedding, score_threshold=score_threshold or self._score_threshold, + group_by=group_by or self._group_by, + group_size=group_size or self._group_size, ) return {"documents": docs} @@ -188,13 +206,16 @@ def __init__( return_embedding: bool = False, filter_policy: Union[str, FilterPolicy] = FilterPolicy.REPLACE, score_threshold: Optional[float] = None, + group_by: Optional[str] = None, + group_size: Optional[int] = None, ): """ Create a QdrantSparseEmbeddingRetriever component. :param document_store: An instance of QdrantDocumentStore. :param filters: A dictionary with filters to narrow down the search space. - :param top_k: The maximum number of documents to retrieve. + :param top_k: The maximum number of documents to retrieve. If using `group_by` parameters, maximum number of + groups to return. :param scale_score: Whether to scale the scores of the retrieved documents or not. :param return_embedding: Whether to return the sparse embedding of the retrieved Documents. :param filter_policy: Policy to determine how filters are applied. Defaults to "replace". @@ -202,6 +223,9 @@ def __init__( Score of the returned result might be higher or smaller than the threshold depending on the Distance function used. E.g. for cosine similarity only higher scores will be returned. + :param group_by: Payload field to group by, must be a string or number field. If the field contains more than 1 + value, all values will be used for grouping. One point can be in multiple groups. + :param group_size: Maximum amount of points to return per group. Default is 3. :raises ValueError: If `document_store` is not an instance of `QdrantDocumentStore`. """ @@ -219,6 +243,8 @@ def __init__( filter_policy if isinstance(filter_policy, FilterPolicy) else FilterPolicy.from_str(filter_policy) ) self._score_threshold = score_threshold + self._group_by = group_by + self._group_size = group_size def to_dict(self) -> Dict[str, Any]: """ @@ -236,6 +262,8 @@ def to_dict(self) -> Dict[str, Any]: filter_policy=self._filter_policy.value, return_embedding=self._return_embedding, score_threshold=self._score_threshold, + group_by=self._group_by, + group_size=self._group_size, ) d["init_parameters"]["document_store"] = self._document_store.to_dict() @@ -268,6 +296,8 @@ def run( scale_score: Optional[bool] = None, return_embedding: Optional[bool] = None, score_threshold: Optional[float] = None, + group_by: Optional[str] = None, + group_size: Optional[int] = None, ): """ Run the Sparse Embedding Retriever on the given input data. @@ -276,13 +306,17 @@ def run( :param filters: Filters applied to the retrieved Documents. The way runtime filters are applied depends on the `filter_policy` chosen at retriever initialization. See init method docstring for more details. - :param top_k: The maximum number of documents to return. + :param top_k: The maximum number of documents to return. If using `group_by` parameters, maximum number of + groups to return. :param scale_score: Whether to scale the scores of the retrieved documents or not. :param return_embedding: Whether to return the embedding of the retrieved Documents. :param score_threshold: A minimal score threshold for the result. Score of the returned result might be higher or smaller than the threshold depending on the Distance function used. E.g. for cosine similarity only higher scores will be returned. + :param group_by: Payload field to group by, must be a string or number field. If the field contains more than 1 + value, all values will be used for grouping. One point can be in multiple groups. + :param group_size: Maximum amount of points to return per group. Default is 3. :returns: The retrieved documents. @@ -296,6 +330,8 @@ def run( scale_score=scale_score or self._scale_score, return_embedding=return_embedding or self._return_embedding, score_threshold=score_threshold or self._score_threshold, + group_by=group_by or self._group_by, + group_size=group_size or self._group_size, ) return {"documents": docs} @@ -342,19 +378,25 @@ def __init__( return_embedding: bool = False, filter_policy: Union[str, FilterPolicy] = FilterPolicy.REPLACE, score_threshold: Optional[float] = None, + group_by: Optional[str] = None, + group_size: Optional[int] = None, ): """ Create a QdrantHybridRetriever component. :param document_store: An instance of QdrantDocumentStore. :param filters: A dictionary with filters to narrow down the search space. - :param top_k: The maximum number of documents to retrieve. + :param top_k: The maximum number of documents to retrieve. If using `group_by` parameters, maximum number of + groups to return. :param return_embedding: Whether to return the embeddings of the retrieved Documents. :param filter_policy: Policy to determine how filters are applied. :param score_threshold: A minimal score threshold for the result. Score of the returned result might be higher or smaller than the threshold depending on the Distance function used. E.g. for cosine similarity only higher scores will be returned. + :param group_by: Payload field to group by, must be a string or number field. If the field contains more than 1 + value, all values will be used for grouping. One point can be in multiple groups. + :param group_size: Maximum amount of points to return per group. Default is 3. :raises ValueError: If 'document_store' is not an instance of QdrantDocumentStore. """ @@ -371,6 +413,8 @@ def __init__( filter_policy if isinstance(filter_policy, FilterPolicy) else FilterPolicy.from_str(filter_policy) ) self._score_threshold = score_threshold + self._group_by = group_by + self._group_size = group_size def to_dict(self) -> Dict[str, Any]: """ @@ -387,6 +431,8 @@ def to_dict(self) -> Dict[str, Any]: filter_policy=self._filter_policy.value, return_embedding=self._return_embedding, score_threshold=self._score_threshold, + group_by=self._group_by, + group_size=self._group_size, ) @classmethod @@ -416,6 +462,8 @@ def run( top_k: Optional[int] = None, return_embedding: Optional[bool] = None, score_threshold: Optional[float] = None, + group_by: Optional[str] = None, + group_size: Optional[int] = None, ): """ Run the Sparse Embedding Retriever on the given input data. @@ -425,12 +473,16 @@ def run( :param filters: Filters applied to the retrieved Documents. The way runtime filters are applied depends on the `filter_policy` chosen at retriever initialization. See init method docstring for more details. - :param top_k: The maximum number of documents to return. + :param top_k: The maximum number of documents to return. If using `group_by` parameters, maximum number of + groups to return. :param return_embedding: Whether to return the embedding of the retrieved Documents. :param score_threshold: A minimal score threshold for the result. Score of the returned result might be higher or smaller than the threshold depending on the Distance function used. E.g. for cosine similarity only higher scores will be returned. + :param group_by: Payload field to group by, must be a string or number field. If the field contains more than 1 + value, all values will be used for grouping. One point can be in multiple groups. + :param group_size: Maximum amount of points to return per group. Default is 3. :returns: The retrieved documents. @@ -444,6 +496,8 @@ def run( top_k=top_k or self._top_k, return_embedding=return_embedding or self._return_embedding, score_threshold=score_threshold or self._score_threshold, + group_by=group_by or self._group_by, + group_size=group_size or self._group_size, ) return {"documents": docs} diff --git a/integrations/qdrant/src/haystack_integrations/document_stores/qdrant/document_store.py b/integrations/qdrant/src/haystack_integrations/document_stores/qdrant/document_store.py index a436fba55..da48e0f28 100644 --- a/integrations/qdrant/src/haystack_integrations/document_stores/qdrant/document_store.py +++ b/integrations/qdrant/src/haystack_integrations/document_stores/qdrant/document_store.py @@ -506,19 +506,25 @@ def _query_by_sparse( scale_score: bool = False, return_embedding: bool = False, score_threshold: Optional[float] = None, + group_by: Optional[str] = None, + group_size: Optional[int] = None, ) -> List[Document]: """ Queries Qdrant using a sparse embedding and returns the most relevant documents. :param query_sparse_embedding: Sparse embedding of the query. :param filters: Filters applied to the retrieved documents. - :param top_k: Maximum number of documents to return. + :param top_k: Maximum number of documents to return. If using `group_by` parameters, maximum number of + groups to return. :param scale_score: Whether to scale the scores of the retrieved documents. :param return_embedding: Whether to return the embeddings of the retrieved documents. :param score_threshold: A minimal score threshold for the result. Score of the returned result might be higher or smaller than the threshold depending on the Distance function used. E.g. for cosine similarity only higher scores will be returned. + :param group_by: Payload field to group by, must be a string or number field. If the field contains more than 1 + value, all values will be used for grouping. One point can be in multiple groups. + :param group_size: Maximum amount of points to return per group. Default is 3. :returns: List of documents that are most similar to `query_sparse_embedding`. @@ -536,22 +542,47 @@ def _query_by_sparse( qdrant_filters = convert_filters_to_qdrant(filters) query_indices = query_sparse_embedding.indices query_values = query_sparse_embedding.values - points = self.client.query_points( - collection_name=self.index, - query=rest.SparseVector( - indices=query_indices, - values=query_values, - ), - using=SPARSE_VECTORS_NAME, - query_filter=qdrant_filters, - limit=top_k, - with_vectors=return_embedding, - score_threshold=score_threshold, - ).points - results = [ - convert_qdrant_point_to_haystack_document(point, use_sparse_embeddings=self.use_sparse_embeddings) - for point in points - ] + if group_by: + groups = self.client.query_points_groups( + collection_name=self.index, + query=rest.SparseVector( + indices=query_indices, + values=query_values, + ), + using=SPARSE_VECTORS_NAME, + query_filter=qdrant_filters, + limit=top_k, + group_by=group_by, + group_size=group_size, + with_vectors=return_embedding, + score_threshold=score_threshold, + ).groups + results = ( + [ + convert_qdrant_point_to_haystack_document(point, use_sparse_embeddings=self.use_sparse_embeddings) + for group in groups + for point in group.hits + ] + if groups + else [] + ) + else: + points = self.client.query_points( + collection_name=self.index, + query=rest.SparseVector( + indices=query_indices, + values=query_values, + ), + using=SPARSE_VECTORS_NAME, + query_filter=qdrant_filters, + limit=top_k, + with_vectors=return_embedding, + score_threshold=score_threshold, + ).points + results = [ + convert_qdrant_point_to_haystack_document(point, use_sparse_embeddings=self.use_sparse_embeddings) + for point in points + ] if scale_score: for document in results: score = document.score @@ -567,37 +598,65 @@ def _query_by_embedding( scale_score: bool = False, return_embedding: bool = False, score_threshold: Optional[float] = None, + group_by: Optional[str] = None, + group_size: Optional[int] = None, ) -> List[Document]: """ Queries Qdrant using a dense embedding and returns the most relevant documents. :param query_embedding: Dense embedding of the query. :param filters: Filters applied to the retrieved documents. - :param top_k: Maximum number of documents to return. + :param top_k: Maximum number of documents to return. If using `group_by` parameters, maximum number of + groups to return. :param scale_score: Whether to scale the scores of the retrieved documents. :param return_embedding: Whether to return the embeddings of the retrieved documents. :param score_threshold: A minimal score threshold for the result. Score of the returned result might be higher or smaller than the threshold depending on the Distance function used. E.g. for cosine similarity only higher scores will be returned. + :param group_by: Payload field to group by, must be a string or number field. If the field contains more than 1 + value, all values will be used for grouping. One point can be in multiple groups. + :param group_size: Maximum amount of points to return per group. Default is 3. :returns: List of documents that are most similar to `query_embedding`. """ qdrant_filters = convert_filters_to_qdrant(filters) + if group_by: + groups = self.client.query_points_groups( + collection_name=self.index, + query=query_embedding, + using=DENSE_VECTORS_NAME if self.use_sparse_embeddings else None, + query_filter=qdrant_filters, + limit=top_k, + group_by=group_by, + group_size=group_size, + with_vectors=return_embedding, + score_threshold=score_threshold, + ).groups + results = ( + [ + convert_qdrant_point_to_haystack_document(point, use_sparse_embeddings=self.use_sparse_embeddings) + for group in groups + for point in group.hits + ] + if groups + else [] + ) + else: + points = self.client.query_points( + collection_name=self.index, + query=query_embedding, + using=DENSE_VECTORS_NAME if self.use_sparse_embeddings else None, + query_filter=qdrant_filters, + limit=top_k, + with_vectors=return_embedding, + score_threshold=score_threshold, + ).points + results = [ + convert_qdrant_point_to_haystack_document(point, use_sparse_embeddings=self.use_sparse_embeddings) + for point in points + ] - points = self.client.query_points( - collection_name=self.index, - query=query_embedding, - using=DENSE_VECTORS_NAME if self.use_sparse_embeddings else None, - query_filter=qdrant_filters, - limit=top_k, - with_vectors=return_embedding, - score_threshold=score_threshold, - ).points - results = [ - convert_qdrant_point_to_haystack_document(point, use_sparse_embeddings=self.use_sparse_embeddings) - for point in points - ] if scale_score: for document in results: score = document.score @@ -616,6 +675,8 @@ def _query_hybrid( top_k: int = 10, return_embedding: bool = False, score_threshold: Optional[float] = None, + group_by: Optional[str] = None, + group_size: Optional[int] = None, ) -> List[Document]: """ Retrieves documents based on dense and sparse embeddings and fuses the results using Reciprocal Rank Fusion. @@ -626,12 +687,16 @@ def _query_hybrid( :param query_embedding: Dense embedding of the query. :param query_sparse_embedding: Sparse embedding of the query. :param filters: Filters applied to the retrieved documents. - :param top_k: Maximum number of documents to return. + :param top_k: Maximum number of documents to return. If using `group_by` parameters, maximum number of + groups to return. :param return_embedding: Whether to return the embeddings of the retrieved documents. :param score_threshold: A minimal score threshold for the result. Score of the returned result might be higher or smaller than the threshold depending on the Distance function used. E.g. for cosine similarity only higher scores will be returned. + :param group_by: Payload field to group by, must be a string or number field. If the field contains more than 1 + value, all values will be used for grouping. One point can be in multiple groups. + :param group_size: Maximum amount of points to return per group. Default is 3. :returns: List of Document that are most similar to `query_embedding` and `query_sparse_embedding`. @@ -651,34 +716,73 @@ def _query_hybrid( qdrant_filters = convert_filters_to_qdrant(filters) try: - points = self.client.query_points( - collection_name=self.index, - prefetch=[ - rest.Prefetch( - query=rest.SparseVector( - indices=query_sparse_embedding.indices, - values=query_sparse_embedding.values, + if group_by: + groups = self.client.query_points_groups( + collection_name=self.index, + prefetch=[ + rest.Prefetch( + query=rest.SparseVector( + indices=query_sparse_embedding.indices, + values=query_sparse_embedding.values, + ), + using=SPARSE_VECTORS_NAME, + filter=qdrant_filters, ), - using=SPARSE_VECTORS_NAME, - filter=qdrant_filters, - ), - rest.Prefetch( - query=query_embedding, - using=DENSE_VECTORS_NAME, - filter=qdrant_filters, - ), - ], - query=rest.FusionQuery(fusion=rest.Fusion.RRF), - limit=top_k, - score_threshold=score_threshold, - with_payload=True, - with_vectors=return_embedding, - ).points + rest.Prefetch( + query=query_embedding, + using=DENSE_VECTORS_NAME, + filter=qdrant_filters, + ), + ], + query=rest.FusionQuery(fusion=rest.Fusion.RRF), + limit=top_k, + group_by=group_by, + group_size=group_size, + score_threshold=score_threshold, + with_payload=True, + with_vectors=return_embedding, + ).groups + else: + points = self.client.query_points( + collection_name=self.index, + prefetch=[ + rest.Prefetch( + query=rest.SparseVector( + indices=query_sparse_embedding.indices, + values=query_sparse_embedding.values, + ), + using=SPARSE_VECTORS_NAME, + filter=qdrant_filters, + ), + rest.Prefetch( + query=query_embedding, + using=DENSE_VECTORS_NAME, + filter=qdrant_filters, + ), + ], + query=rest.FusionQuery(fusion=rest.Fusion.RRF), + limit=top_k, + score_threshold=score_threshold, + with_payload=True, + with_vectors=return_embedding, + ).points + except Exception as e: msg = "Error during hybrid search" raise QdrantStoreError(msg) from e - results = [convert_qdrant_point_to_haystack_document(point, use_sparse_embeddings=True) for point in points] + if group_by: + results = ( + [ + convert_qdrant_point_to_haystack_document(point, use_sparse_embeddings=self.use_sparse_embeddings) + for group in groups + for point in group.hits + ] + if groups + else [] + ) + else: + results = [convert_qdrant_point_to_haystack_document(point, use_sparse_embeddings=True) for point in points] return results diff --git a/integrations/qdrant/tests/test_document_store.py b/integrations/qdrant/tests/test_document_store.py index 112b7e5ac..79523531b 100644 --- a/integrations/qdrant/tests/test_document_store.py +++ b/integrations/qdrant/tests/test_document_store.py @@ -97,6 +97,39 @@ def test_query_hybrid(self, generate_sparse_embedding): assert document.sparse_embedding assert document.embedding + def test_query_hybrid_with_group_by(self, generate_sparse_embedding): + document_store = QdrantDocumentStore(location=":memory:", use_sparse_embeddings=True) + + docs = [] + for i in range(20): + docs.append( + Document( + content=f"doc {i}", + sparse_embedding=generate_sparse_embedding(), + embedding=_random_embeddings(768), + meta={"group_field": i // 2}, + ) + ) + + document_store.write_documents(docs) + + sparse_embedding = SparseEmbedding(indices=[0, 1, 2, 3], values=[0.1, 0.8, 0.05, 0.33]) + embedding = [0.1] * 768 + + results: List[Document] = document_store._query_hybrid( + query_sparse_embedding=sparse_embedding, + query_embedding=embedding, + top_k=3, + return_embedding=True, + group_by="meta.group_field", + group_size=2, + ) + assert len(results) == 6 + + for document in results: + assert document.sparse_embedding + assert document.embedding + def test_query_hybrid_fail_without_sparse_embedding(self, document_store): sparse_embedding = SparseEmbedding(indices=[0, 1, 2, 3], values=[0.1, 0.8, 0.05, 0.33]) embedding = [0.1] * 768 diff --git a/integrations/qdrant/tests/test_retriever.py b/integrations/qdrant/tests/test_retriever.py index eb0386828..bd6b92842 100644 --- a/integrations/qdrant/tests/test_retriever.py +++ b/integrations/qdrant/tests/test_retriever.py @@ -27,6 +27,8 @@ def test_init_default(self): assert retriever._filter_policy == FilterPolicy.REPLACE assert retriever._return_embedding is False assert retriever._score_threshold is None + assert retriever._group_by is None + assert retriever._group_size is None retriever = QdrantEmbeddingRetriever(document_store=document_store, filter_policy="replace") assert retriever._filter_policy == FilterPolicy.REPLACE @@ -87,6 +89,8 @@ def test_to_dict(self): "scale_score": False, "return_embedding": False, "score_threshold": None, + "group_by": None, + "group_size": None, }, } @@ -104,6 +108,8 @@ def test_from_dict(self): "scale_score": False, "return_embedding": True, "score_threshold": None, + "group_by": None, + "group_size": None, }, } retriever = QdrantEmbeddingRetriever.from_dict(data) @@ -115,6 +121,8 @@ def test_from_dict(self): assert retriever._scale_score is False assert retriever._return_embedding is True assert retriever._score_threshold is None + assert retriever._group_by is None + assert retriever._group_size is None def test_run(self, filterable_docs: List[Document]): document_store = QdrantDocumentStore(location=":memory:", index="Boi", use_sparse_embeddings=False) @@ -200,6 +208,26 @@ def test_run_with_sparse_activated(self, filterable_docs: List[Document]): for document in results: assert document.embedding is None + def test_run_with_group_by(self, filterable_docs: List[Document]): + document_store = QdrantDocumentStore(location=":memory:", index="Boi", use_sparse_embeddings=True) + # Add group_field metadata to documents + for index, doc in enumerate(filterable_docs): + doc.meta = {"group_field": index // 2} # So at least two docs have same group each time + document_store.write_documents(filterable_docs) + + retriever = QdrantEmbeddingRetriever(document_store=document_store) + results = retriever.run( + query_embedding=_random_embeddings(768), + top_k=3, + return_embedding=False, + group_by="meta.group_field", + group_size=2, + )["documents"] + assert len(results) >= 3 # This test is Flaky + assert len(results) <= 6 # This test is Flaky + for document in results: + assert document.embedding is None + class TestQdrantSparseEmbeddingRetriever(FilterableDocsFixtureMixin): def test_init_default(self): @@ -211,6 +239,8 @@ def test_init_default(self): assert retriever._filter_policy == FilterPolicy.REPLACE assert retriever._return_embedding is False assert retriever._score_threshold is None + assert retriever._group_by is None + assert retriever._group_size is None retriever = QdrantSparseEmbeddingRetriever(document_store=document_store, filter_policy="replace") assert retriever._filter_policy == FilterPolicy.REPLACE @@ -271,6 +301,8 @@ def test_to_dict(self): "return_embedding": False, "filter_policy": "replace", "score_threshold": None, + "group_by": None, + "group_size": None, }, } @@ -288,6 +320,8 @@ def test_from_dict(self): "return_embedding": True, "filter_policy": "replace", "score_threshold": None, + "group_by": None, + "group_size": None, }, } retriever = QdrantSparseEmbeddingRetriever.from_dict(data) @@ -299,6 +333,8 @@ def test_from_dict(self): assert retriever._scale_score is False assert retriever._return_embedding is True assert retriever._score_threshold is None + assert retriever._group_by is None + assert retriever._group_size is None def test_from_dict_no_filter_policy(self): data = { @@ -313,6 +349,8 @@ def test_from_dict_no_filter_policy(self): "scale_score": False, "return_embedding": True, "score_threshold": None, + "group_by": None, + "group_size": None, }, } retriever = QdrantSparseEmbeddingRetriever.from_dict(data) @@ -324,6 +362,8 @@ def test_from_dict_no_filter_policy(self): assert retriever._scale_score is False assert retriever._return_embedding is True assert retriever._score_threshold is None + assert retriever._group_by is None + assert retriever._group_size is None def test_run(self, filterable_docs: List[Document], generate_sparse_embedding): document_store = QdrantDocumentStore(location=":memory:", index="Boi", use_sparse_embeddings=True) @@ -345,6 +385,29 @@ def test_run(self, filterable_docs: List[Document], generate_sparse_embedding): for document in results: assert document.sparse_embedding + def test_run_with_group_by(self, filterable_docs: List[Document], generate_sparse_embedding): + document_store = QdrantDocumentStore(location=":memory:", index="Boi", use_sparse_embeddings=True) + + # Add fake sparse embedding to documents + for index, doc in enumerate(filterable_docs): + doc.sparse_embedding = generate_sparse_embedding() + doc.meta = {"group_field": index // 2} # So at least two docs have same group each time + document_store.write_documents(filterable_docs) + retriever = QdrantSparseEmbeddingRetriever(document_store=document_store) + sparse_embedding = SparseEmbedding(indices=[0, 1, 2, 3], values=[0.1, 0.8, 0.05, 0.33]) + results = retriever.run( + query_sparse_embedding=sparse_embedding, + top_k=3, + return_embedding=True, + group_by="meta.group_field", + group_size=2, + )["documents"] + assert len(results) >= 3 # This test is Flaky + assert len(results) <= 6 # This test is Flaky + + for document in results: + assert document.sparse_embedding + class TestQdrantHybridRetriever: def test_init_default(self): @@ -357,6 +420,8 @@ def test_init_default(self): assert retriever._filter_policy == FilterPolicy.REPLACE assert retriever._return_embedding is False assert retriever._score_threshold is None + assert retriever._group_by is None + assert retriever._group_size is None retriever = QdrantHybridRetriever(document_store=document_store, filter_policy="replace") assert retriever._filter_policy == FilterPolicy.REPLACE @@ -416,6 +481,8 @@ def test_to_dict(self): "filter_policy": "replace", "return_embedding": True, "score_threshold": None, + "group_by": None, + "group_size": None, }, } @@ -432,6 +499,8 @@ def test_from_dict(self): "filter_policy": "replace", "return_embedding": True, "score_threshold": None, + "group_by": None, + "group_size": None, }, } retriever = QdrantHybridRetriever.from_dict(data) @@ -442,6 +511,8 @@ def test_from_dict(self): assert retriever._filter_policy == FilterPolicy.REPLACE assert retriever._return_embedding assert retriever._score_threshold is None + assert retriever._group_by is None + assert retriever._group_size is None def test_from_dict_no_filter_policy(self): data = { @@ -455,6 +526,8 @@ def test_from_dict_no_filter_policy(self): "top_k": 5, "return_embedding": True, "score_threshold": None, + "group_by": None, + "group_size": None, }, } retriever = QdrantHybridRetriever.from_dict(data) @@ -465,6 +538,8 @@ def test_from_dict_no_filter_policy(self): assert retriever._filter_policy == FilterPolicy.REPLACE # defaults to REPLACE assert retriever._return_embedding assert retriever._score_threshold is None + assert retriever._group_by is None + assert retriever._group_size is None def test_run(self): mock_store = Mock(spec=QdrantDocumentStore) @@ -488,3 +563,31 @@ def test_run(self): assert res["documents"][0].content == "Test doc" assert res["documents"][0].embedding == [0.1, 0.2] assert res["documents"][0].sparse_embedding == sparse_embedding + + def test_run_with_group_by(self): + mock_store = Mock(spec=QdrantDocumentStore) + sparse_embedding = SparseEmbedding(indices=[0, 1, 2, 3], values=[0.1, 0.8, 0.05, 0.33]) + mock_store._query_hybrid.return_value = [ + Document(content="Test doc", embedding=[0.1, 0.2], sparse_embedding=sparse_embedding) + ] + + retriever = QdrantHybridRetriever(document_store=mock_store) + res = retriever.run( + query_embedding=[0.5, 0.7], + query_sparse_embedding=SparseEmbedding(indices=[0, 5], values=[0.1, 0.7]), + group_by="meta.group_field", + group_size=2, + ) + + call_args = mock_store._query_hybrid.call_args + assert call_args[1]["query_embedding"] == [0.5, 0.7] + assert call_args[1]["query_sparse_embedding"].indices == [0, 5] + assert call_args[1]["query_sparse_embedding"].values == [0.1, 0.7] + assert call_args[1]["top_k"] == 10 + assert call_args[1]["return_embedding"] is False + assert call_args[1]["group_by"] == "meta.group_field" + assert call_args[1]["group_size"] == 2 + + assert res["documents"][0].content == "Test doc" + assert res["documents"][0].embedding == [0.1, 0.2] + assert res["documents"][0].sparse_embedding == sparse_embedding diff --git a/integrations/ragas/CHANGELOG.md b/integrations/ragas/CHANGELOG.md index 7055f1931..94946bddc 100644 --- a/integrations/ragas/CHANGELOG.md +++ b/integrations/ragas/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## [integrations/ragas-v1.0.1] - 2024-09-11 + +### ๐Ÿ› Bug Fixes + +- Add upper-bound pin to `ragas` dependency in `ragas-haystack` (#1076) + +### ๐Ÿงช Testing + +- Do not retry tests in `hatch run test` command (#954) + ## [integrations/ragas-v1.0.0] - 2024-07-24 ### โš™๏ธ Miscellaneous Tasks diff --git a/integrations/ragas/pyproject.toml b/integrations/ragas/pyproject.toml index edc33eee1..d9ae6ca02 100644 --- a/integrations/ragas/pyproject.toml +++ b/integrations/ragas/pyproject.toml @@ -22,7 +22,7 @@ classifiers = [ "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ] -dependencies = ["haystack-ai", "ragas>=0.1.11"] +dependencies = ["haystack-ai", "ragas>=0.1.11,<=0.1.16"] [project.urls] Source = "https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/ragas" @@ -41,7 +41,13 @@ root = "../.." git_describe_command = 'git describe --tags --match="integrations/ragas-v[0-9]*"' [tool.hatch.envs.default] -dependencies = ["coverage[toml]>=6.5", "pytest", "pytest-rerunfailures", "haystack-pydoc-tools", "pytest-asyncio"] +dependencies = [ + "coverage[toml]>=6.5", + "pytest", + "pytest-rerunfailures", + "haystack-pydoc-tools", + "pytest-asyncio", +] [tool.hatch.envs.default.scripts] test = "pytest {args:tests}" test-cov = "coverage run -m pytest {args:tests}" diff --git a/integrations/unstructured/src/haystack_integrations/components/converters/unstructured/converter.py b/integrations/unstructured/src/haystack_integrations/components/converters/unstructured/converter.py index 637c0840f..9230ecb0d 100644 --- a/integrations/unstructured/src/haystack_integrations/components/converters/unstructured/converter.py +++ b/integrations/unstructured/src/haystack_integrations/components/converters/unstructured/converter.py @@ -27,7 +27,7 @@ class UnstructuredFileConverter: A component for converting files to Haystack Documents using the Unstructured API (hosted or running locally). For the supported file types and the specific API parameters, see - [Unstructured docs](https://unstructured-io.github.io/unstructured/api.html). + [Unstructured docs](https://docs.unstructured.io/api-reference/api-services/overview). Usage example: ```python @@ -68,7 +68,7 @@ def __init__( :param separator: Separator between elements when concatenating them into one text field. :param unstructured_kwargs: Additional parameters that are passed to the Unstructured API. For the available parameters, see - [Unstructured API docs](https://unstructured-io.github.io/unstructured/apis/api_parameters.html). + [Unstructured API docs](https://docs.unstructured.io/api-reference/api-services/api-parameters). :param progress_bar: Whether to show a progress bar during the conversion. """ diff --git a/integrations/weaviate/CHANGELOG.md b/integrations/weaviate/CHANGELOG.md index bddde1b7d..dacf3fef8 100644 --- a/integrations/weaviate/CHANGELOG.md +++ b/integrations/weaviate/CHANGELOG.md @@ -1,6 +1,12 @@ # Changelog -## [unreleased] +## [integrations/weaviate-v3.0.0] - 2024-09-12 + +### โš™๏ธ Miscellaneous Tasks + +- Weaviate - remove legacy filter support (#1070) + +## [integrations/weaviate-v2.2.1] - 2024-09-07 ### ๐Ÿš€ Features @@ -10,6 +16,12 @@ - Weaviate filter error (#811) - Fix connection to Weaviate Cloud Service (#624) +- Pin weaviate-client (#1046) +- Weaviate - fix connection issues with some WCS URLs (#1058) + +### ๐Ÿงช Testing + +- Do not retry tests in `hatch run test` command (#954) ### โš™๏ธ Miscellaneous Tasks diff --git a/integrations/weaviate/pyproject.toml b/integrations/weaviate/pyproject.toml index 4f9a3245a..624a06f1d 100644 --- a/integrations/weaviate/pyproject.toml +++ b/integrations/weaviate/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ ] dependencies = [ "haystack-ai", - "weaviate-client", + "weaviate-client>=4.0", "haystack-pydoc-tools", "python-dateutil", ] diff --git a/integrations/weaviate/src/haystack_integrations/document_stores/weaviate/document_store.py b/integrations/weaviate/src/haystack_integrations/document_stores/weaviate/document_store.py index 82088dd89..e312b1473 100644 --- a/integrations/weaviate/src/haystack_integrations/document_stores/weaviate/document_store.py +++ b/integrations/weaviate/src/haystack_integrations/document_stores/weaviate/document_store.py @@ -12,7 +12,6 @@ from haystack.dataclasses.document import Document from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError from haystack.document_stores.types.policy import DuplicatePolicy -from haystack.utils.filters import convert import weaviate from weaviate.collections.classes.data import DataObject @@ -68,7 +67,7 @@ class WeaviateDocumentStore: from haystack_integrations.document_stores.weaviate.auth import AuthApiKey from haystack_integrations.document_stores.weaviate.document_store import WeaviateDocumentStore - os.environ["WEAVIATE_API_KEY"] = "MY_API_KEY + os.environ["WEAVIATE_API_KEY"] = "MY_API_KEY" document_store = WeaviateDocumentStore( url="rAnD0mD1g1t5.something.weaviate.cloud", @@ -172,17 +171,18 @@ def client(self): if self._client: return self._client - if self._url and self._url.startswith("http") and self._url.endswith(".weaviate.network"): - # We use this utility function instead of using WeaviateClient directly like in other cases - # otherwise we'd have to parse the URL to get some information about the connection. - # This utility function does all that for us. - self._client = weaviate.connect_to_wcs( + if self._url and self._url.endswith((".weaviate.network", ".weaviate.cloud")): + # If we detect that the URL is a Weaviate Cloud URL, we use the utility function to connect + # instead of using WeaviateClient directly like in other cases. + # Among other things, the utility function takes care of parsing the URL. + self._client = weaviate.connect_to_weaviate_cloud( self._url, auth_credentials=self._auth_client_secret.resolve_value() if self._auth_client_secret else None, headers=self._additional_headers, additional_config=self._additional_config, ) else: + # Embedded, local Docker deployment or custom connection. # proxies, timeout_config, trust_env are part of additional_config now # startup_period has been removed self._client = weaviate.WeaviateClient( @@ -387,7 +387,8 @@ def filter_documents(self, filters: Optional[Dict[str, Any]] = None) -> List[Doc :returns: A list of Documents that match the given filters. """ if filters and "operator" not in filters and "conditions" not in filters: - filters = convert(filters) + msg = "Invalid filter syntax. See https://docs.haystack.deepset.ai/docs/metadata-filtering for details." + raise ValueError(msg) result = [] if filters: diff --git a/integrations/weaviate/tests/test_document_store.py b/integrations/weaviate/tests/test_document_store.py index 8d531cade..190c23408 100644 --- a/integrations/weaviate/tests/test_document_store.py +++ b/integrations/weaviate/tests/test_document_store.py @@ -660,15 +660,6 @@ def test_embedding_retrieval_with_distance_and_certainty(self, document_store): with pytest.raises(ValueError): document_store._embedding_retrieval(query_embedding=[], distance=0.1, certainty=0.1) - def test_filter_documents_with_legacy_filters(self, document_store): - docs = [] - for index in range(10): - docs.append(Document(content="This is some content", meta={"index": index})) - document_store.write_documents(docs) - result = document_store.filter_documents({"content": {"$eq": "This is some content"}}) - - assert len(result) == 10 - def test_filter_documents_below_default_limit(self, document_store): docs = [] for index in range(9998):