-
Notifications
You must be signed in to change notification settings - Fork 126
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'main' into remove-error-pypi-docs
- Loading branch information
Showing
8 changed files
with
431 additions
and
9 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| # to make use of the JinaReaderConnector, we first need to install the Haystack integration | ||
| # pip install jina-haystack | ||
|
|
||
| # then we must set the JINA_API_KEY environment variable | ||
| # export JINA_API_KEY=<your-api-key> | ||
|
|
||
|
|
||
| from haystack_integrations.components.connectors.jina import JinaReaderConnector | ||
|
|
||
| # we can use the JinaReaderConnector to process a URL and return the textual content of the page | ||
| reader = JinaReaderConnector(mode="read") | ||
| query = "https://example.com" | ||
| result = reader.run(query=query) | ||
|
|
||
| print(result) | ||
| # {'documents': [Document(id=fa3e51e4ca91828086dca4f359b6e1ea2881e358f83b41b53c84616cb0b2f7cf, | ||
| # content: 'This domain is for use in illustrative examples in documents. You may use this domain in literature ...', | ||
| # meta: {'title': 'Example Domain', 'description': '', 'url': 'https://example.com/', 'usage': {'tokens': 42}})]} | ||
|
|
||
|
|
||
| # we can perform a web search by setting the mode to "search" | ||
| reader = JinaReaderConnector(mode="search") | ||
| query = "UEFA Champions League 2024" | ||
| result = reader.run(query=query) | ||
|
|
||
| print(result) | ||
| # {'documents': Document(id=6a71abf9955594232037321a476d39a835c0cb7bc575d886ee0087c973c95940, | ||
| # content: '2024/25 UEFA Champions League: Matches, draw, final, key dates | UEFA Champions League | UEFA.com...', | ||
| # meta: {'title': '2024/25 UEFA Champions League: Matches, draw, final, key dates', | ||
| # 'description': 'What are the match dates? Where is the 2025 final? How will the competition work?', | ||
| # 'url': 'https://www.uefa.com/uefachampionsleague/news/...', | ||
| # 'usage': {'tokens': 5581}}), ...]} | ||
|
|
||
|
|
||
| # finally, we can perform fact-checking by setting the mode to "ground" (experimental) | ||
| reader = JinaReaderConnector(mode="ground") | ||
| query = "ChatGPT was launched in 2017" | ||
| result = reader.run(query=query) | ||
|
|
||
| print(result) | ||
| # {'documents': [Document(id=f0c964dbc1ebb2d6584c8032b657150b9aa6e421f714cc1b9f8093a159127f0c, | ||
| # content: 'The statement that ChatGPT was launched in 2017 is incorrect. Multiple references confirm that ChatG...', | ||
| # meta: {'factuality': 0, 'result': False, 'references': [ | ||
| # {'url': 'https://en.wikipedia.org/wiki/ChatGPT', | ||
| # 'keyQuote': 'ChatGPT is a generative artificial intelligence (AI) chatbot developed by OpenAI and launched in 2022.', | ||
| # 'isSupportive': False}, ...], | ||
| # 'usage': {'tokens': 10188}})]} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
7 changes: 7 additions & 0 deletions
7
integrations/jina/src/haystack_integrations/components/connectors/jina/__init__.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| # SPDX-FileCopyrightText: 2023-present deepset GmbH <[email protected]> | ||
| # | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
| from .reader import JinaReaderConnector | ||
| from .reader_mode import JinaReaderMode | ||
|
|
||
| __all__ = ["JinaReaderConnector", "JinaReaderMode"] |
141 changes: 141 additions & 0 deletions
141
integrations/jina/src/haystack_integrations/components/connectors/jina/reader.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,141 @@ | ||
| # SPDX-FileCopyrightText: 2023-present deepset GmbH <[email protected]> | ||
| # | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| import json | ||
| from typing import Any, Dict, List, Optional, Union | ||
| from urllib.parse import quote | ||
|
|
||
| import requests | ||
| from haystack import Document, component, default_from_dict, default_to_dict | ||
| from haystack.utils import Secret, deserialize_secrets_inplace | ||
|
|
||
| from .reader_mode import JinaReaderMode | ||
|
|
||
| READER_ENDPOINT_URL_BY_MODE = { | ||
| JinaReaderMode.READ: "https://r.jina.ai/", | ||
| JinaReaderMode.SEARCH: "https://s.jina.ai/", | ||
| JinaReaderMode.GROUND: "https://g.jina.ai/", | ||
| } | ||
|
|
||
|
|
||
| @component | ||
| class JinaReaderConnector: | ||
| """ | ||
| A component that interacts with Jina AI's reader service to process queries and return documents. | ||
| This component supports different modes of operation: `read`, `search`, and `ground`. | ||
| Usage example: | ||
| ```python | ||
| from haystack_integrations.components.connectors.jina import JinaReaderConnector | ||
| reader = JinaReaderConnector(mode="read") | ||
| query = "https://example.com" | ||
| result = reader.run(query=query) | ||
| document = result["documents"][0] | ||
| print(document.content) | ||
| >>> "This domain is for use in illustrative examples..." | ||
| ``` | ||
| """ | ||
|
|
||
| def __init__( | ||
| self, | ||
| mode: Union[JinaReaderMode, str], | ||
| api_key: Secret = Secret.from_env_var("JINA_API_KEY"), # noqa: B008 | ||
| json_response: bool = True, | ||
| ): | ||
| """ | ||
| Initialize a JinaReader instance. | ||
| :param mode: The operation mode for the reader (`read`, `search` or `ground`). | ||
| - `read`: process a URL and return the textual content of the page. | ||
| - `search`: search the web and return textual content of the most relevant pages. | ||
| - `ground`: call the grounding engine to perform fact checking. | ||
| For more information on the modes, see the [Jina Reader documentation](https://jina.ai/reader/). | ||
| :param api_key: The Jina API key. It can be explicitly provided or automatically read from the | ||
| environment variable JINA_API_KEY (recommended). | ||
| :param json_response: Controls the response format from the Jina Reader API. | ||
| If `True`, requests a JSON response, resulting in Documents with rich structured metadata. | ||
| If `False`, requests a raw response, resulting in one Document with minimal metadata. | ||
| """ | ||
| self.api_key = api_key | ||
| self.json_response = json_response | ||
|
|
||
| if isinstance(mode, str): | ||
| mode = JinaReaderMode.from_str(mode) | ||
| self.mode = mode | ||
|
|
||
| def to_dict(self) -> Dict[str, Any]: | ||
| """ | ||
| Serializes the component to a dictionary. | ||
| :returns: | ||
| Dictionary with serialized data. | ||
| """ | ||
| return default_to_dict( | ||
| self, | ||
| api_key=self.api_key.to_dict(), | ||
| mode=str(self.mode), | ||
| json_response=self.json_response, | ||
| ) | ||
|
|
||
| @classmethod | ||
| def from_dict(cls, data: Dict[str, Any]) -> "JinaReaderConnector": | ||
| """ | ||
| Deserializes the component from a dictionary. | ||
| :param data: | ||
| Dictionary to deserialize from. | ||
| :returns: | ||
| Deserialized component. | ||
| """ | ||
| deserialize_secrets_inplace(data["init_parameters"], keys=["api_key"]) | ||
| return default_from_dict(cls, data) | ||
|
|
||
| def _json_to_document(self, data: dict) -> Document: | ||
| """ | ||
| Convert a JSON response/record to a Document, depending on the reader mode. | ||
| """ | ||
| if self.mode == JinaReaderMode.GROUND: | ||
| content = data.pop("reason") | ||
| else: | ||
| content = data.pop("content") | ||
| document = Document(content=content, meta=data) | ||
| return document | ||
|
|
||
| @component.output_types(document=List[Document]) | ||
| def run(self, query: str, headers: Optional[Dict[str, str]] = None): | ||
| """ | ||
| Process the query/URL using the Jina AI reader service. | ||
| :param query: The query string or URL to process. | ||
| :param headers: Optional headers to include in the request for customization. Refer to the | ||
| [Jina Reader documentation](https://jina.ai/reader/) for more information. | ||
| :returns: | ||
| A dictionary with the following keys: | ||
| - `documents`: A list of `Document` objects. | ||
| """ | ||
| headers = headers or {} | ||
| headers["Authorization"] = f"Bearer {self.api_key.resolve_value()}" | ||
|
|
||
| if self.json_response: | ||
| headers["Accept"] = "application/json" | ||
|
|
||
| endpoint_url = READER_ENDPOINT_URL_BY_MODE[self.mode] | ||
| encoded_target = quote(query, safe="") | ||
| url = f"{endpoint_url}{encoded_target}" | ||
|
|
||
| response = requests.get(url, headers=headers, timeout=60) | ||
|
|
||
| # raw response: we just return a single Document with text | ||
| if not self.json_response: | ||
| meta = {"content_type": response.headers["Content-Type"], "query": query} | ||
| return {"documents": [Document(content=response.content, meta=meta)]} | ||
|
|
||
| response_json = json.loads(response.content).get("data", {}) | ||
| if self.mode == JinaReaderMode.SEARCH: | ||
| documents = [self._json_to_document(record) for record in response_json] | ||
| return {"documents": documents} | ||
|
|
||
| return {"documents": [self._json_to_document(response_json)]} |
40 changes: 40 additions & 0 deletions
40
integrations/jina/src/haystack_integrations/components/connectors/jina/reader_mode.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| # SPDX-FileCopyrightText: 2023-present deepset GmbH <[email protected]> | ||
| # | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
| from enum import Enum | ||
|
|
||
|
|
||
| class JinaReaderMode(Enum): | ||
| """ | ||
| Enum representing modes for the Jina Reader. | ||
| Modes: | ||
| READ: Process a URL and return the textual content of the page. | ||
| SEARCH: Search the web and return the textual content of the most relevant pages. | ||
| GROUND: Call the grounding engine to perform fact checking. | ||
| """ | ||
|
|
||
| READ = "read" | ||
| SEARCH = "search" | ||
| GROUND = "ground" | ||
|
|
||
| def __str__(self): | ||
| return self.value | ||
|
|
||
| @classmethod | ||
| def from_str(cls, string: str) -> "JinaReaderMode": | ||
| """ | ||
| Create the reader mode from a string. | ||
| :param string: | ||
| String to convert. | ||
| :returns: | ||
| Reader mode. | ||
| """ | ||
| enum_map = {e.value: e for e in JinaReaderMode} | ||
| reader_mode = enum_map.get(string) | ||
| if reader_mode is None: | ||
| msg = f"Unknown reader mode '{string}'. Supported modes are: {list(enum_map.keys())}" | ||
| raise ValueError(msg) | ||
| return reader_mode |
Oops, something went wrong.