diff --git a/README.md b/README.md index 6d99fb00..4274f7f4 100644 --- a/README.md +++ b/README.md @@ -34,23 +34,21 @@ that includes it. Once it reaches the end of its lifespan, the experiment will b ## Experiments catalog -The latest version of the package contains the following experiments: - -| Name | Type | Expected End Date | Dependencies | Cookbook | Discussion | -| --------------------------- | -------------------------- | ---------------------------- | ------------ | -------- | ---------- | -| [`EvaluationHarness`][1] | Evaluation orchestrator | October 2024 | None | Open In Colab | [Discuss](https://github.com/deepset-ai/haystack-experimental/discussions/74) | -| [`OpenAIFunctionCaller`][2] | Function Calling Component | October 2024 | None | ๐Ÿ”œ | | -| [`OpenAPITool`][3] | OpenAPITool component | October 2024 | jsonref | Open In Colab | [Discuss](https://github.com/deepset-ai/haystack-experimental/discussions/79)| -| Support for Tools: [refactored `ChatMessage` dataclass][10], [`Tool` dataclass][4], [refactored `OpenAIChatGenerator`][11], [refactored `OllamaChatGenerator`][14], [refactored `HuggingFaceAPIChatGenerator`][15], [refactored `AnthropicChatGenerator`][16], [`ToolInvoker` component][12] | Tool Calling support | November 2024 | jsonschema | Open In Colab | [Discuss](https://github.com/deepset-ai/haystack-experimental/discussions/98)| -| [`ChatMessageWriter`][5] | Memory Component | December 2024 | None | Open In Colab | [Discuss](https://github.com/deepset-ai/haystack-experimental/discussions/75) | -| [`ChatMessageRetriever`][6] | Memory Component | December 2024 | None | Open In Colab | [Discuss](https://github.com/deepset-ai/haystack-experimental/discussions/75) | -| [`InMemoryChatMessageStore`][7] | Memory Store | December 2024 | None | Open In Colab | [Discuss](https://github.com/deepset-ai/haystack-experimental/discussions/75) | -| [`Auto-Merging Retriever`][8] & [`HierarchicalDocumentSplitter`][9]| Document Splitting & Retrieval Technique | December 2024 | None | Open In Colab | [Discuss](https://github.com/deepset-ai/haystack-experimental/discussions/78) | -| [`LLMMetadataExtractor`][13] | Metadata extraction with LLM | December 2024 | None | ๐Ÿ”œ | [Discuss](https://github.com/deepset-ai/haystack-experimental/discussions/130) | +### Active experiments + +| Name | Type | Expected End Date | Dependencies | Cookbook | Discussion | +| -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------- | ----------------- | ------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------ | +| [`EvaluationHarness`][1] | Evaluation orchestrator | October 2024 | None | Open In Colab | [Discuss](https://github.com/deepset-ai/haystack-experimental/discussions/74) | +| Support for Tools: [refactored `ChatMessage` dataclass][10], [`Tool` dataclass][4], [refactored `OpenAIChatGenerator`][11], [refactored `OllamaChatGenerator`][14], [refactored `HuggingFaceAPIChatGenerator`][15], [refactored `AnthropicChatGenerator`][16], [`ToolInvoker` component][12] | Tool Calling support | November 2024 | jsonschema | Open In Colab | [Discuss](https://github.com/deepset-ai/haystack-experimental/discussions/98) | +| [`ChatMessageWriter`][5] | Memory Component | December 2024 | None | Open In Colab | [Discuss](https://github.com/deepset-ai/haystack-experimental/discussions/75) | +| [`ChatMessageRetriever`][6] | Memory Component | December 2024 | None | Open In Colab | [Discuss](https://github.com/deepset-ai/haystack-experimental/discussions/75) | +| [`InMemoryChatMessageStore`][7] | Memory Store | December 2024 | None | Open In Colab | [Discuss](https://github.com/deepset-ai/haystack-experimental/discussions/75) | +| [`Auto-Merging Retriever`][8] & [`HierarchicalDocumentSplitter`][9] | Document Splitting & Retrieval Technique | December 2024 | None | Open In Colab | [Discuss](https://github.com/deepset-ai/haystack-experimental/discussions/78) | +| [`LLMMetadataExtractor`][13] | Metadata extraction with LLM | December 2024 | None | ๐Ÿ”œ | [Discuss](https://github.com/deepset-ai/haystack-experimental/discussions/130) | [1]: https://github.com/deepset-ai/haystack-experimental/tree/main/haystack_experimental/evaluation/harness -[2]: https://github.com/deepset-ai/haystack-experimental/tree/main/haystack_experimental/components/tools/openai -[3]: https://github.com/deepset-ai/haystack-experimental/tree/main/haystack_experimental/components/tools/openapi +[2]: https://github.com/deepset-ai/haystack-experimental/tree/fe20b69b31243f8a3976e4661d9aa8c88a2847d2/haystack_experimental/components/tools/openai +[3]: https://github.com/deepset-ai/haystack-experimental/tree/fe20b69b31243f8a3976e4661d9aa8c88a2847d2/haystack_experimental/components/tools/openapi [4]: https://github.com/deepset-ai/haystack-experimental/tree/main/haystack_experimental/dataclasses/tool.py [5]: https://github.com/deepset-ai/haystack-experimental/blob/main/haystack_experimental/components/writers/chat_message_writer.py [6]: https://github.com/deepset-ai/haystack-experimental/blob/main/haystack_experimental/components/retrievers/chat_message_retriever.py @@ -65,6 +63,13 @@ The latest version of the package contains the following experiments: [15]: https://github.com/deepset-ai/haystack-experimental/blob/main/haystack_experimental/components/generators/chat/hugging_face_api.py [16]: https://github.com/deepset-ai/haystack-experimental/blob/main/haystack_experimental/components/generators/anthropic/chat/chat_generator.py +### Discontinued experiments + +| Name | Type | Final release | Cookbook | +| --------------------------- | -------------------------- | ------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | +| [`OpenAIFunctionCaller`][2] | Function Calling Component | 0.3.0 | None | +| [`OpenAPITool`][3] | OpenAPITool component | 0.3.0 | [Notebook](https://github.com/deepset-ai/haystack-experimental/blob/fe20b69b31243f8a3976e4661d9aa8c88a2847d2/examples/openapitool.ipynb) | + ## Usage Experimental new features can be imported like any other Haystack integration package: diff --git a/docs/pydoc/config/tools_api.yml b/docs/pydoc/config/tools_api.yml index f3a9e3b2..b9818557 100644 --- a/docs/pydoc/config/tools_api.yml +++ b/docs/pydoc/config/tools_api.yml @@ -1,12 +1,7 @@ loaders: - type: haystack_pydoc_tools.loaders.CustomPythonLoader search_path: [../../../] - modules: - [ - "haystack_experimental.components.tools.tool_invoker", - "haystack_experimental.components.tools.openai.function_caller", - "haystack_experimental.components.tools.openapi.openapi_tool" - ] + modules: ["haystack_experimental.components.tools.tool_invoker"] ignore_when_discovered: ["__init__"] processors: - type: filter diff --git a/examples/openapitool.ipynb b/examples/openapitool.ipynb deleted file mode 100644 index 0847c960..00000000 --- a/examples/openapitool.ipynb +++ /dev/null @@ -1,572 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "HCnOg0poyvSo" - }, - "source": [ - "# ๐Ÿงช Invoking APIs with `OpenAPITool`\n", - "\n", - "\"Open\n", - "\n", - "Many APIs available on the Web provide an OpenAPI specification that describes their structure and syntax.\n", - "\n", - "[`OpenAPITool`](https://docs.haystack.deepset.ai/reference/openapi-api) is an experimental Haystack component that allows you to call an API using payloads generated from human instructions.\n", - "\n", - "Here's a brief overview of how it works:\n", - "- At initialization, it loads the OpenAPI specification from a URL or a file.\n", - "- At runtime:\n", - " - Converts human instructions into a suitable API payload using a Chat Language Model (LLM).\n", - " - Invokes the API.\n", - " - Returns the API response, wrapped in a Chat Message.\n", - "\n", - "Let's see this component in action..." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ruIf93lVLaO9" - }, - "source": [ - "## Setup" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "XvsPuqZcaIvp" - }, - "outputs": [], - "source": [ - "! pip install haystack-ai jsonref" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ew6t1MY1LkP7" - }, - "source": [ - "In this notebook, we will be using some APIs that require an API key. Let's set them as environment variables." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "eOz5Ry4IaOub" - }, - "outputs": [], - "source": [ - "import os\n", - "\n", - "os.environ[\"OPENAI_API_KEY\"]=\"...\"\n", - "\n", - "# free API key: https://www.firecrawl.dev/\n", - "os.environ[\"FIRECRAWL_API_KEY\"]=\"...\"\n", - "\n", - "# free API key: https://serper.dev/\n", - "os.environ[\"SERPERDEV_API_KEY\"]=\"...\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "8NUEM_uVfpdi" - }, - "source": [ - "## Call an API without credentials\n", - "\n", - "In the first example, we use Open-Meteo, a Free Weather API that does not require authentication.\n", - "\n", - "We use `OPENAI` as LLM provider. Other supported providers are `ANTHROPIC` and `COHERE`." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "MDy213SLXZQy", - "outputId": "4e9e1607-4212-4ce7-e409-ded548f83ab0" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'service_response': [ChatMessage(content='{\"latitude\": 37.763283, \"longitude\": -122.41286, \"generationtime_ms\": 0.07700920104980469, \"utc_offset_seconds\": -25200, \"timezone\": \"America/Los_Angeles\", \"timezone_abbreviation\": \"PDT\", \"elevation\": 18.0, \"current_weather_units\": {\"time\": \"iso8601\", \"interval\": \"seconds\", \"temperature\": \"\\\\u00b0C\", \"windspeed\": \"km/h\", \"winddirection\": \"\\\\u00b0\", \"is_day\": \"\", \"weathercode\": \"wmo code\"}, \"current_weather\": {\"time\": \"2024-07-29T06:45\", \"interval\": 900, \"temperature\": 13.2, \"windspeed\": 12.7, \"winddirection\": 263, \"is_day\": 1, \"weathercode\": 45}}', role=, name=None, meta={})]}" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from haystack.dataclasses import ChatMessage\n", - "from haystack_experimental.components.tools.openapi import OpenAPITool, LLMProvider\n", - "from haystack.utils import Secret\n", - "\n", - "tool = OpenAPITool(generator_api=LLMProvider.OPENAI,\n", - " spec=\"https://raw.githubusercontent.com/open-meteo/open-meteo/main/openapi.yml\")\n", - "\n", - "tool.run(messages=[ChatMessage.from_user(\"Weather in San Francisco, US\")])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "EQmvkszfgZoi" - }, - "source": [ - "## Incorporate `OpenAPITool` in a Pipeline\n", - "\n", - "Next, let's create a simple Pipeline where the service response is translated into a human-understandable format using the Language Model.\n", - "\n", - "We use a [`ChatPromptBuilder`](https://docs.haystack.deepset.ai/docs/chatpromptbuilder) to create a list of Chat Messages for the LM." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "xTRcny9ig-0t", - "outputId": "d5f21778-b4fc-4eb1-8e3a-dbc6f99f9b44" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "\n", - "๐Ÿš… Components\n", - " - meteo: OpenAPITool\n", - " - builder: ChatPromptBuilder\n", - " - llm: OpenAIChatGenerator\n", - "๐Ÿ›ค๏ธ Connections\n", - " - meteo.service_response -> builder.service_response (List[ChatMessage])\n", - " - builder.prompt -> llm.messages (List[ChatMessage])" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from typing import List\n", - "from haystack import Pipeline\n", - "from haystack.components.builders import ChatPromptBuilder\n", - "from haystack.components.generators.chat import OpenAIChatGenerator\n", - "\n", - "messages = [ChatMessage.from_user(\"{{user_message}}\"), ChatMessage.from_user(\"{{service_response}}\")]\n", - "builder = ChatPromptBuilder(template=messages)\n", - "\n", - "pipe = Pipeline()\n", - "pipe.add_component(\"meteo\", tool)\n", - "pipe.add_component(\"builder\", builder)\n", - "pipe.add_component(\"llm\", OpenAIChatGenerator(generation_kwargs={\"max_tokens\": 1024}))\n", - "\n", - "pipe.connect(\"meteo\", \"builder.service_response\")\n", - "pipe.connect(\"builder\", \"llm.messages\")" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": { - "id": "6yF2RyfQhAaV" - }, - "outputs": [], - "source": [ - "result = pipe.run(data={\"meteo\": {\"messages\": [ChatMessage.from_user(\"weather in San Francisco, US\")]},\n", - " \"builder\": {\"user_message\": [ChatMessage.from_user(\"Explain the weather in San Francisco in a human understandable way\")]}})" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "RJg3P-k2hY5M", - "outputId": "938ca3f1-6c57-4969-8d56-ed9cab9258af" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The current weather in San Francisco is 13.2ยฐC with a windspeed of 12.7 km/h coming from the west-southwest. It is currently daytime and the weather code indicates that it is partly cloudy.\n" - ] - } - ], - "source": [ - "print(result[\"llm\"][\"replies\"][0].content)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "2r5h2f0ehfrb" - }, - "source": [ - "## Use an API with credentials in a Pipeline\n", - "\n", - "In this example, we use [Firecrawl](https://www.firecrawl.dev/): a project that scrape Web pages (and Web sites) and convert them into clean text. Firecrawl has an API that requires an API key.\n", - "\n", - "In the following Pipeline, we use Firecrawl to scrape a news article, which is then summarized using a Language Model." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Y64kS9RbatCQ", - "outputId": "1db016c0-981f-4252-ce67-50bb8aae264e" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "\n", - "๐Ÿš… Components\n", - " - firecrawl: OpenAPITool\n", - " - builder: ChatPromptBuilder\n", - " - llm: OpenAIChatGenerator\n", - "๐Ÿ›ค๏ธ Connections\n", - " - firecrawl.service_response -> builder.service_response (List[ChatMessage])\n", - " - builder.prompt -> llm.messages (List[ChatMessage])" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "messages = [ChatMessage.from_user(\"{{user_message}}\"), ChatMessage.from_user(\"{{service_response}}\")]\n", - "builder = ChatPromptBuilder(template=messages)\n", - "\n", - "\n", - "pipe = Pipeline()\n", - "pipe.add_component(\"firecrawl\", OpenAPITool(generator_api=LLMProvider.OPENAI,\n", - " spec=\"https://raw.githubusercontent.com/mendableai/firecrawl/main/apps/api/openapi.json\",\n", - " credentials=Secret.from_env_var(\"FIRECRAWL_API_KEY\")))\n", - "pipe.add_component(\"builder\", builder)\n", - "pipe.add_component(\"llm\", OpenAIChatGenerator(generation_kwargs={\"max_tokens\": 1024}))\n", - "\n", - "pipe.connect(\"firecrawl\", \"builder.service_response\")\n", - "pipe.connect(\"builder\", \"llm.messages\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "LSj6l6_6bRjT" - }, - "outputs": [], - "source": [ - "user_prompt = \"Given the article below, list the most important facts in a bulleted list. Do not include repetitions. Max 5 points.\"\n", - "\n", - "result = pipe.run(data={\"firecrawl\": {\"messages\": [ChatMessage.from_user(\"Scrape https://lite.cnn.com/2024/07/18/style/rome-ancient-papal-palace/index.html\")]},\n", - " \"builder\": {\"user_message\": [ChatMessage.from_user(user_prompt)]}})" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "c_TJ8R2VbWMM", - "outputId": "ac0128f3-a2c0-413b-8052-24f6730c6e4f" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "- Remains of a medieval palace believed to be where popes lived before the Vatican have been excavated in Rome.\n", - "- The site is located near the Archbasilica of St John Lateran in Rome.\n", - "- The building's initial structure dates back to Emperor Constantine in the 4th century and was expanded between the 9th and 13th centuries.\n", - "- The papacy resided in the palace until 1305 when it temporarily moved to Avignon in France.\n", - "- The discovery was made ahead of renovations for the 2025 Catholic Holy Year, expected to attract over 30 million pilgrims and tourists to Rome.\n" - ] - } - ], - "source": [ - "print(result[\"llm\"][\"replies\"][0].content)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Wv6weSs3iQ40" - }, - "source": [ - "## Create a Pipeline with multiple `OpenAPITool` components\n", - "\n", - "In this example, we show a Pipeline where multiple alternative APIs can be invoked depending on the user query. In particular, a Google Search (via Serper.dev) can be performed or a single page can be scraped using Firecrawl.\n", - "\n", - "โš ๏ธ The approach shown is just one way to achieve this using [conditional routing](https://docs.haystack.deepset.ai/docs/conditionalrouter). We are currently experimenting with tool support in Haystack, and there may be simpler ways to achieve the same result in the future." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "id": "73ypFwhtKbH9" - }, - "outputs": [], - "source": [ - "import json\n", - "\n", - "decision_prompt_template = \"\"\"\n", - "You are a virtual assistant, equipped with the following tools:\n", - "\n", - "- `{\"tool_name\": \"search_web\", \"tool_description\": \"Access to Google search, use this tool whenever information on recents events is needed\"}`\n", - "- `{\"tool_name\": \"scrape_page\", \"tool_description\": \"Use this tool to scrape and crawl web pages\"}`\n", - "\n", - "Select the most appropriate tool to resolve the user's query. Respond in JSON format, specifying the user request and the chosen tool for the response.\n", - "If you can't match user query to an above listed tools, respond with `none`.\n", - "\n", - "\n", - "######\n", - "Here are some examples:\n", - "\n", - "```json\n", - "{\n", - " \"query\": \"Why did Elon Musk recently sue OpenAI?\",\n", - " \"response\": \"search_web\"\n", - "}\n", - "{\n", - " \"query\": \"What is on the front-page of hackernews today?\",\n", - " \"response\": \"scrape_page\"\n", - "}\n", - "{\n", - " \"query\": \"Tell me about Berlin\",\n", - " \"response\": \"none\"\n", - "}\n", - "```\n", - "\n", - "Choose the best tool (or none) for each user request, considering the current context of the conversation specified above.\n", - "\n", - "{\"query\": {{query}}, \"response\": }\n", - "\"\"\"\n", - "\n", - "def get_tool_name(replies):\n", - " try:\n", - " tool_name = json.loads(replies)[\"response\"]\n", - " return tool_name\n", - " except:\n", - " return \"error\"\n", - "\n", - "\n", - "routes = [\n", - " {\n", - " \"condition\": \"{{replies[0] | get_tool_name == 'search_web'}}\",\n", - " \"output\": \"{{query}}\",\n", - " \"output_name\": \"search_web\",\n", - " \"output_type\": str,\n", - " },\n", - " {\n", - " \"condition\": \"{{replies[0] | get_tool_name == 'scrape_page'}}\",\n", - " \"output\": \"{{query}}\",\n", - " \"output_name\": \"scrape_page\",\n", - " \"output_type\": str,\n", - " },\n", - " {\n", - " \"condition\": \"{{replies[0] | get_tool_name == 'none'}}\",\n", - " \"output\": \"{{replies[0]}}\",\n", - " \"output_name\": \"no_tools\",\n", - " \"output_type\": str,\n", - " },\n", - " {\n", - " \"condition\": \"{{replies[0] | get_tool_name == 'error'}}\",\n", - " \"output\": \"{{replies[0]}}\",\n", - " \"output_name\": \"error\",\n", - " \"output_type\": str,\n", - " },\n", - "]" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "0BCqEIsmNXJx", - "outputId": "cc3bcfa7-56b1-4f41-d38f-8b4f4dbcece9" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "\n", - "๐Ÿš… Components\n", - " - prompt_builder: PromptBuilder\n", - " - llm: OpenAIGenerator\n", - " - router: ConditionalRouter\n", - " - search_web_chat_builder: ChatPromptBuilder\n", - " - scrape_page_chat_builder: ChatPromptBuilder\n", - " - search_web_tool: OpenAPITool\n", - " - scrape_page_tool: OpenAPITool\n", - "๐Ÿ›ค๏ธ Connections\n", - " - prompt_builder.prompt -> llm.prompt (str)\n", - " - llm.replies -> router.replies (List[str])\n", - " - router.search_web -> search_web_chat_builder.query (str)\n", - " - router.scrape_page -> scrape_page_chat_builder.query (str)\n", - " - search_web_chat_builder.prompt -> search_web_tool.messages (List[ChatMessage])\n", - " - scrape_page_chat_builder.prompt -> scrape_page_tool.messages (List[ChatMessage])" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from haystack.components.builders import PromptBuilder, ChatPromptBuilder\n", - "from haystack.components.routers import ConditionalRouter\n", - "from haystack.components.generators import OpenAIGenerator\n", - "\n", - "\n", - "messages = [ChatMessage.from_user(\"{{query}}\")]\n", - "\n", - "search_web_chat_builder = ChatPromptBuilder(template=messages)\n", - "scrape_page_chat_builder = ChatPromptBuilder(template=messages)\n", - "\n", - "search_web_tool = OpenAPITool(generator_api=LLMProvider.OPENAI,\n", - " spec=\"https://bit.ly/serper_dev_spec_yaml\",\n", - " credentials=Secret.from_env_var(\"SERPERDEV_API_KEY\"))\n", - "\n", - "scrape_page_tool = OpenAPITool(generator_api=LLMProvider.OPENAI,\n", - " spec=\"https://raw.githubusercontent.com/mendableai/firecrawl/main/apps/api/openapi.json\",\n", - " credentials=Secret.from_env_var(\"FIRECRAWL_API_KEY\"))\n", - "\n", - "pipe = Pipeline()\n", - "pipe.add_component(\"prompt_builder\", PromptBuilder(template=decision_prompt_template))\n", - "pipe.add_component(\"llm\", OpenAIGenerator())\n", - "pipe.add_component(\"router\", ConditionalRouter(routes, custom_filters={\"get_tool_name\": get_tool_name}))\n", - "pipe.add_component(\"search_web_chat_builder\", search_web_chat_builder)\n", - "pipe.add_component(\"scrape_page_chat_builder\", scrape_page_chat_builder)\n", - "pipe.add_component(\"search_web_tool\", search_web_tool)\n", - "pipe.add_component(\"scrape_page_tool\", scrape_page_tool)\n", - "\n", - "pipe.connect(\"prompt_builder\", \"llm\")\n", - "pipe.connect(\"llm.replies\", \"router.replies\")\n", - "pipe.connect(\"router.search_web\", \"search_web_chat_builder\")\n", - "pipe.connect(\"router.scrape_page\", \"scrape_page_chat_builder\")\n", - "pipe.connect(\"search_web_chat_builder\", \"search_web_tool\")\n", - "pipe.connect(\"scrape_page_chat_builder\", \"scrape_page_tool\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "93qfiQQqjNEy", - "outputId": "aefa654a-a8f9-43ae-a4d8-7fdc46d7910a" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'llm': {'meta': [{'model': 'gpt-3.5-turbo-0125',\n", - " 'index': 0,\n", - " 'finish_reason': 'stop',\n", - " 'usage': {'completion_tokens': 23,\n", - " 'prompt_tokens': 248,\n", - " 'total_tokens': 271}}]},\n", - " 'search_web_tool': {'service_response': [ChatMessage(content='{\"searchParameters\": {\"q\": \"UEFA European Football Championship winner\", \"type\": \"search\", \"engine\": \"google\"}, \"answerBox\": {\"title\": \"Spain national football teamUEFA European Championship / Latest Champion\", \"answer\": \"Spain national football team\"}, \"organic\": [{\"title\": \"UEFA European Championship - Wikipedia\", \"link\": \"https://en.wikipedia.org/wiki/UEFA_European_Championship\", \"snippet\": \"The most recent championship, held in Germany in 2024, was won by Spain, who lifted a record fourth European title after beating England 2\\\\u20131 in the final at the ...\", \"sitelinks\": [{\"title\": \"Finals\", \"link\": \"https://en.wikipedia.org/wiki/List_of_UEFA_European_Championship_finals\"}, {\"title\": \"European Championship in\", \"link\": \"https://en.wikipedia.org/wiki/European_Championship_in_football\"}, {\"title\": \"UEFA Women\\'s Championship\", \"link\": \"https://en.wikipedia.org/wiki/UEFA_Women%27s_Championship\"}, {\"title\": \"Euro 2024\", \"link\": \"https://en.wikipedia.org/wiki/UEFA_Euro_2024\"}], \"position\": 1}, {\"title\": \"Winners List of the UEFA European Championship - The Euros\", \"link\": \"https://www.topendsports.com/events/soccer/uefa-euros/winners.htm\", \"snippet\": \"Ten different countries have won the tournament: Spain have won four times, Germany has three titles, France and Italy with two titles while Portugal, ...\", \"position\": 2}, {\"title\": \"List of UEFA European Championship finals - Wikipedia\", \"link\": \"https://en.wikipedia.org/wiki/List_of_UEFA_European_Championship_finals\", \"snippet\": \"The winners of the first ever final, held in Paris in 1960, were the Soviet Union, who defeated Yugoslavia 2\\\\u20131 after extra time, while in the latest one, hosted ...\", \"sitelinks\": [{\"title\": \"History\", \"link\": \"https://en.wikipedia.org/wiki/List_of_UEFA_European_Championship_finals#History\"}, {\"title\": \"List of finals\", \"link\": \"https://en.wikipedia.org/wiki/List_of_UEFA_European_Championship_finals#List_of_finals\"}, {\"title\": \"Results by nation\", \"link\": \"https://en.wikipedia.org/wiki/List_of_UEFA_European_Championship_finals#Results_by_nation\"}], \"position\": 3}, {\"title\": \"UEFA Euro Winners List from 1960 to today - Marca.com\", \"link\": \"https://www.marca.com/en/football/uefa-euro/winners.html\", \"snippet\": \"Check the updated ranking of all the winners of the UEFA Euro Cup year by year. Record of European Championships throughout history in Marca English.\", \"sitelinks\": [{\"title\": \"Euro 2024 Live Scores\", \"link\": \"https://www.marca.com/en/scores/football/uefa-euro.html\"}, {\"title\": \"Euro 2024 Schedule\", \"link\": \"https://www.marca.com/en/football/uefa-euro/schedule.html\"}, {\"title\": \"Euro Stadiums Germany 2024\", \"link\": \"https://www.marca.com/en/football/uefa-euro/stadiums.html\"}], \"position\": 4}, {\"title\": \"Most titles | History | UEFA EURO\", \"link\": \"https://www.uefa.com/uefaeuro/history/winners/\", \"snippet\": \"View the official UEFA EURO winners list at UEFA.com. Find out which teams have lifted the most trophies since the competition began.\", \"sitelinks\": [{\"title\": \"Portugal 1-0 France\", \"link\": \"https://www.uefa.com/uefaeuro/match/2017907--portugal-vs-france/\"}, {\"title\": \"2012: Spain 4-0 Italy\", \"link\": \"https://www.uefa.com/uefaeuro/match/2003351--spain-vs-italy/\"}, {\"title\": \"West Germany vs USSR\", \"link\": \"https://www.uefa.com/uefaeuro/match/3838--west-germany-vs-ussr/\"}, {\"title\": \"Spain (1964)\", \"link\": \"https://www.uefa.com/uefaeuro/match/3996--spain-vs-ussr/\"}], \"position\": 5}, {\"title\": \"European Championship | History, Winners, & Facts | Britannica\", \"link\": \"https://www.britannica.com/sports/European-Championship\", \"snippet\": \"... football tournaments. Learn more about the European Championship, including its winners ... Also known as: Euro, European Nation\\'s Cup, UEFA European Championship.\", \"position\": 6}, {\"title\": \"UEFA European Championship News, Stats, Scores - ESPN\", \"link\": \"https://www.espn.com/soccer/league/_/name/uefa.euro\", \"snippet\": \"Follow all the latest UEFA European Championship football news, fixtures, stats, and more on ESPN.\", \"position\": 7}, {\"title\": \"UEFA Euro winners: Know the champions - full list\", \"link\": \"https://olympics.com/en/news/uefa-european-championships-euro-winners-list-champions\", \"snippet\": \"Know all the UEFA European Championship winners. The Soviet Union won the first title in 1960 while Spain won the UEFA Euro 2024.\", \"date\": \"Jul 11, 2021\", \"position\": 8}, {\"title\": \"History | UEFA EURO\", \"link\": \"https://www.uefa.com/uefaeuro/history/\", \"snippet\": \"Official UEFA EURO history. Season-by-season guide, extensive all-time stats, plus video highlights of every final to date.\", \"sitelinks\": [{\"title\": \"Most titles\", \"link\": \"https://www.uefa.com/uefaeuro/history/winners/\"}, {\"title\": \"UEFA European...\", \"link\": \"https://www.uefa.com/uefaeuro/history/news/0253-0d81c56ff408-45bf000cd5b6-1000--uefa-european-championship-roll-of-honour/\"}, {\"title\": \"2020\", \"link\": \"https://www.uefa.com/uefaeuro/history/seasons/2020/\"}, {\"title\": \"All-time stats\", \"link\": \"https://www.uefa.com/uefaeuro/history/rankings/\"}], \"position\": 9}, {\"title\": \"UEFA EURO all-time winners 2024 - Statista\", \"link\": \"https://www.statista.com/statistics/378217/uefa-euro-titles-winners-and-finalists/\", \"snippet\": \"La Roja most recently won the competition in 2024, defeating England 2-1 in the EURO 2024 final. Read more. Countries with the most men\\'s UEFA ...\", \"date\": \"6 days ago\", \"position\": 10}], \"peopleAlsoAsk\": [{\"question\": \"Who won UEFA European Championship?\", \"snippet\": \"Spain national football team\\\\nUEFA European Championship / Latest Champion\", \"title\": \"\"}, {\"question\": \"Who is the winner of European Champions League?\", \"snippet\": \"Real Madrid CF\\\\nUEFA Champions League / Latest Champion\", \"title\": \"\"}, {\"question\": \"How many countries have won the European football Championship?\", \"snippet\": \"Final tournament Map of countries\\' best results. 10 countries have won, counting Germany and West Germany as one.\", \"title\": \"UEFA European Championship - Wikipedia\", \"link\": \"https://en.wikipedia.org/wiki/UEFA_European_Championship\"}, {\"question\": \"Who is the current women\\'s European champion?\", \"snippet\": \"The competition is the women\\'s equivalent of the UEFA European Championship. The reigning champions are England, who won their home tournament in 2022. The most successful nation in the history of the tournament is Germany, with eight titles.\", \"title\": \"UEFA Women\\'s Championship - Wikipedia\", \"link\": \"https://en.wikipedia.org/wiki/UEFA_Women%27s_Championship\"}], \"relatedSearches\": [{\"query\": \"UEFA Euro 2020 Final\"}, {\"query\": \"Uefa european football championship winner list\"}, {\"query\": \"Most euro cup winners list\"}, {\"query\": \"Last Euro Cup winners\"}, {\"query\": \"Uefa european football championship winner 2021\"}, {\"query\": \"Euro Cup winners list men\\'s\"}, {\"query\": \"Euro winners list since 2000\"}, {\"query\": \"Euro Cup winners list 2024\"}, {\"query\": \"Next Euro Cup 2024\"}]}', role=, name=None, meta={})]}}" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "query = \"Who won the UEFA European Football Championship?\"\n", - "\n", - "pipe.run({\"prompt_builder\": {\"query\": query}, \"router\": {\"query\": query}})" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "T2nC2yXGipRS", - "outputId": "c347781a-839a-4d04-8f66-d03101701a8e" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'llm': {'meta': [{'model': 'gpt-3.5-turbo-0125',\n", - " 'index': 0,\n", - " 'finish_reason': 'stop',\n", - " 'usage': {'completion_tokens': 26,\n", - " 'prompt_tokens': 250,\n", - " 'total_tokens': 276}}]},\n", - " 'scrape_page_tool': {'service_response': [ChatMessage(content='{\"success\": true, \"data\": {\"content\": \"\\\\n\\\\n[British Broadcasting Corporation](/)\\\\n\\\\n[Watch Live](/watch-live-news)\\\\n\\\\nRegisterSign In\\\\n\\\\n* [Home](/)\\\\n \\\\n* [News](/news)\\\\n \\\\n* [Sport](/sport)\\\\n \\\\n* [Business](/business)\\\\n \\\\n* [Innovation](/innovation)\\\\n \\\\n* [Culture](/culture)\\\\n \\\\n* [Travel](/travel)\\\\n \\\\n* [Earth](/future-planet)\\\\n \\\\n* [Video](/video)\\\\n \\\\n* [Live](/live)\\\\n \\\\n\\\\nRegisterSign In\\\\n\\\\n[Home](/)\\\\n\\\\nNews\\\\n\\\\n[Sport](/sport)\\\\n\\\\nBusiness\\\\n\\\\nInnovation\\\\n\\\\nCulture\\\\n\\\\nTravel\\\\n\\\\nEarth\\\\n\\\\n[Video](/video)\\\\n\\\\nLive\\\\n\\\\n[Audio](https://www.bbc.co.uk/sounds)\\\\n\\\\n[Weather](https://www.bbc.com/weather)\\\\n\\\\n[Newsletters](https://www.bbc.com/newsletters)\\\\n\\\\n[![Biden\\'s last public appearance in Delaware before announcing he would withdraw from the race](https://ichef.bbci.co.uk/news/480/cpsprodpb/05a8/live/0dce30e0-47ee-11ef-96a8-e710c6bfc866.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nThe last days of the Biden campaign \\\\u2013 BBC correspondent\\\\u2019s account\\\\\\\\\\\\n-----------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nBBC correspondent Tom Bateman takes us behind the scenes in the final days of Biden\\'s campaign.\\\\\\\\\\\\n\\\\\\\\\\\\n3 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nUS & Canada](/news/articles/c250zqgrpqgo)\\\\n\\\\n[![Joe Biden looks on during a speech at the White House on 14 July ](https://ichef.bbci.co.uk/news/480/cpsprodpb/ffdd/live/f2086770-4808-11ef-96a8-e710c6bfc866.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nIsolating at a beach house, Biden gave aides one minute notice of exit\\\\\\\\\\\\n----------------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nMost of the president\\'s senior aides found out about his decision to exit the race minutes before he publicly announced it.\\\\\\\\\\\\n\\\\\\\\\\\\n15 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nUS & Canada](/news/articles/c3gdzmdje5xo)\\\\n\\\\n[![Biden\\'s last public appearance in Delaware before announcing he would withdraw from the race](https://ichef.bbci.co.uk/news/480/cpsprodpb/05a8/live/0dce30e0-47ee-11ef-96a8-e710c6bfc866.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nThe last days of the Biden campaign \\\\u2013 BBC correspondent\\\\u2019s account\\\\\\\\\\\\n-----------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nBBC correspondent Tom Bateman takes us behind the scenes in the final days of Biden\\'s campaign.\\\\\\\\\\\\n\\\\\\\\\\\\n3 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nUS & Canada](/news/articles/c250zqgrpqgo)\\\\n\\\\n[![Joe Biden looks on during a speech at the White House on 14 July ](https://ichef.bbci.co.uk/news/480/cpsprodpb/ffdd/live/f2086770-4808-11ef-96a8-e710c6bfc866.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nIsolating at a beach house, Biden gave aides one minute notice of exit\\\\\\\\\\\\n----------------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nMost of the president\\'s senior aides found out about his decision to exit the race minutes before he publicly announced it.\\\\\\\\\\\\n\\\\\\\\\\\\n15 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nUS & Canada](/news/articles/c3gdzmdje5xo)\\\\n\\\\n[![Kamala Harris gestures as she speaks at the White House on Monday](https://ichef.bbci.co.uk/ace/standard/480/cpsprodpb/7856/live/1ce684a0-4844-11ef-9e1c-3b4a473456a6.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nLIVE\\\\\\\\\\\\n\\\\\\\\\\\\nKamala Harris speaks for first time since Biden left race - as endorsements mount\\\\\\\\\\\\n---------------------------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nThe US vice-president appears at the White House for a pre-scheduled event - as key Democrats line up to back her candidacy.](https://www.bbc.com/news/live/cv2gryx1yx1t)\\\\n\\\\n* * *\\\\n\\\\n[What Biden quitting means for Harris, the Democrats and Trump\\\\\\\\\\\\n-------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nPresident Biden has upended the 2024 White House race for the Democrats. Here is what it means for Kamala Harris, his party and Trump.\\\\\\\\\\\\n\\\\\\\\\\\\n19 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nUS & Canada](/news/articles/cpwd8yzw45qo)\\\\n\\\\n[Who could be Kamala Harris\\'s running mate?\\\\\\\\\\\\n------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nIt\\\\u2019s not a done deal but some potential rivals have quickly thrown their support behind her.\\\\\\\\\\\\n\\\\\\\\\\\\n10 mins ago\\\\\\\\\\\\n\\\\\\\\\\\\nUS & Canada](/news/articles/c80ekdwk9zro)\\\\n\\\\n[LIVE\\\\\\\\\\\\n\\\\\\\\\\\\nTensions flare as Congress presses Secret Service boss on \\'failed\\' Trump rally security\\\\\\\\\\\\n---------------------------------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nMembers of both parties have called for Kimberly Cheatle to resign in a House committee hearing that is seeking answers over security failures at the rally on 13 July.](https://www.bbc.com/news/live/c724wqpy4qnt)\\\\n\\\\n[The president\\'s protectors are hardly noticeable - until things go wrong\\\\\\\\\\\\n------------------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nThe attempted assassination of Donald Trump has raised questions about the Secret Service\\'s record.\\\\\\\\\\\\n\\\\\\\\\\\\n6 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nUS & Canada](/news/articles/c16j896003xo)\\\\n\\\\n* * *\\\\n\\\\nOnly from the BBC\\\\n-----------------\\\\n\\\\n[![Two people sit in deck chairs surrounded by water (Credit: Getty Images)](credit: Getty Images)\\\\\\\\\\\\n\\\\\\\\\\\\nWhy you are probably sitting for too long\\\\\\\\\\\\n-----------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nSitting down is ingrained in most peoples\\' days. But staying sedentary for too long can increase the risk of serious health conditions like cardiovascular disease and type 2 diabetes.\\\\\\\\\\\\n\\\\\\\\\\\\n4 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nFuture](/future/article/20240722-why-you-are-probably-sitting-down-for-too-long)\\\\n\\\\n[![South African convicted killer Louis van Schoor stares into the middle distance](https://ichef.bbci.co.uk/news/480/cpsprodpb/229c/live/a8aaa380-4520-11ef-b74c-bb483a802c97.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nMass killer who \\\\u2018hunted\\\\u2019 black people says police encouraged him\\\\\\\\\\\\n----------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nEx-security guard, Louis van Schoor, killed dozens in South Africa but was only jailed for seven murders.\\\\\\\\\\\\n\\\\\\\\\\\\n17 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nAfrica](/news/articles/c51yqdy3q61o)\\\\n\\\\n* * *\\\\n\\\\n[More news\\\\\\\\\\\\n---------](https://www.bbc.com/news)\\\\n\\\\u00a0\\\\n\\\\n[![File photo showing downtown Dubai\\'s skyline (12 June 2021)](https://www.bbc.com/12%20June%202021)\\\\\\\\\\\\n\\\\\\\\\\\\nUAE jails 57 Bangladeshis over protests against own government\\\\\\\\\\\\n--------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nThree defendants were sentenced to life after being convicted of \\\\\"inciting riots\\\\\" in the Gulf state.\\\\\\\\\\\\n\\\\\\\\\\\\n5 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nWorld](/news/articles/crgk8gnpg0zo)\\\\n\\\\n[![A Palestinian woman sitting on a wheelchair is pulled by a man as they flee eastern Khan Younis in response to an Israeli evacuation order, in the southern Gaza Strip (22 July 2024)](https://www.bbc.com/22%20July%202024)\\\\\\\\\\\\n\\\\\\\\\\\\nIsrael orders evacuation of part of Gaza humanitarian zone\\\\\\\\\\\\n----------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nStrikes are reported near Khan Younis, after people are told to leave eastern areas in the zone.\\\\\\\\\\\\n\\\\\\\\\\\\n2 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nMiddle East](/news/articles/cgerz8we1vgo)\\\\n\\\\n[![Black and white photo of Prince George in shirt and suit](https://ichef.bbci.co.uk/news/480/cpsprodpb/d483/live/f8c3d070-480a-11ef-9e1c-3b4a473456a6.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nNew Prince George photo released on 11th birthday\\\\\\\\\\\\n-------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nKensington Palace has posted the image, taken by his mother, Catherine, Princess of Wales.\\\\\\\\\\\\n\\\\\\\\\\\\n7 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nUK](/news/articles/cw4yxd3dw7qo)\\\\n\\\\n[Piastri wins in Hungary after Norris team orders row\\\\\\\\\\\\n----------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nOscar Piastri takes his maiden grand prix victory ahead of McLaren team-mate Lando Norris in a dramatic race in Hungary.\\\\\\\\\\\\n\\\\\\\\\\\\n1 day ago\\\\\\\\\\\\n\\\\\\\\\\\\nFormula 1](/sport/formula1/articles/cg3jzy3e8q1o)\\\\n\\\\n[India alert after boy dies from Nipah virus in Kerala\\\\\\\\\\\\n-----------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nThe virus is transmitted from animals such as pigs and fruit bats to humans.\\\\\\\\\\\\n\\\\\\\\\\\\n6 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nAsia](/news/articles/cj50d7e9vp6o)\\\\n\\\\n[![A Palestinian woman sitting on a wheelchair is pulled by a man as they flee eastern Khan Younis in response to an Israeli evacuation order, in the southern Gaza Strip (22 July 2024)](https://www.bbc.com/22%20July%202024)\\\\\\\\\\\\n\\\\\\\\\\\\nIsrael orders evacuation of part of Gaza humanitarian zone\\\\\\\\\\\\n----------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nStrikes are reported near Khan Younis, after people are told to leave eastern areas in the zone.\\\\\\\\\\\\n\\\\\\\\\\\\n2 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nMiddle East](/news/articles/cgerz8we1vgo)\\\\n\\\\n[![A Kanwariya carries holy water collected from Ganga River in Haridwar during Kanwar Yatra at Sector 14A, on July 10, 2023 in Noida](https://ichef.bbci.co.uk/news/480/cpsprodpb/b9f0/live/7390b470-47e9-11ef-80f2-8f08f27244b7.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nIndia court blocks order for eateries to display owners\\' names\\\\\\\\\\\\n--------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nCritics say ordering restaurants to prominently display names of owners is discriminatory towards Muslims.\\\\\\\\\\\\n\\\\\\\\\\\\n6 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nAsia](/news/articles/czrj18yp489o)\\\\n\\\\n[![Information screen informs train travellers of global IT outage. ](https://ichef.bbci.co.uk/news/480/cpsprodpb/bb5b/live/363718b0-47c3-11ef-be99-e9774e2b4d6b.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\n\\'Significant number\\' of devices fixed - CrowdStrike\\\\\\\\\\\\n---------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nCybersecurity firm behind global outage says it continues to focus on restoring all impacted computers.\\\\\\\\\\\\n\\\\\\\\\\\\n13 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nBusiness](/news/articles/cgl7e33n1d0o)\\\\n\\\\n[![An employee uses a jigsaw to cut a plastic pipe at the Grundfos AS factory in Chennai, India, on Monday, Nov. 27, 2017.](https://ichef.bbci.co.uk/news/480/cpsprodpb/7f64/live/70d23da0-45ad-11ef-b74c-bb483a802c97.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nModi\\'s new budget faces jobs crisis test in India\\\\\\\\\\\\n-------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nMr Modi\\'s biggest challenge in his third term will be bridging the rich-poor divide.\\\\\\\\\\\\n\\\\\\\\\\\\n16 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nAsia](/news/articles/cq5jwyel12qo)\\\\n\\\\n[![Black and white photo of Prince George in shirt and suit](https://ichef.bbci.co.uk/news/480/cpsprodpb/d483/live/f8c3d070-480a-11ef-9e1c-3b4a473456a6.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nNew Prince George photo released on 11th birthday\\\\\\\\\\\\n-------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nKensington Palace has posted the image, taken by his mother, Catherine, Princess of Wales.\\\\\\\\\\\\n\\\\\\\\\\\\n7 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nUK](/news/articles/cw4yxd3dw7qo)\\\\n\\\\n* * *\\\\n\\\\nMust watch\\\\n----------\\\\n\\\\n[![Biden speaks at a rally](https://ichef.bbci.co.uk/news/480/cpsprodpb/cc3c/live/cd687d10-4808-11ef-b74c-bb483a802c97.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nBiden\\\\u2019s disastrous few weeks... in 90 seconds\\\\\\\\\\\\n---------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nPresident Biden has faced intense pressure to step aside since his faltering debate performance.\\\\\\\\\\\\n\\\\\\\\\\\\n17 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nUS & Canada](/news/videos/cx028eq4qg1o)\\\\n\\\\n[![Royal Netherlands Navy\\'s anti-submarine helicopter](https://ichef.bbci.co.uk/news/480/cpsprodpb/80df/live/5d6dff00-482e-11ef-9e1c-3b4a473456a6.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nInside the Netherlands Navy\\'s anti-submarine helicopter\\\\\\\\\\\\n-------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nThe NH90 was on display at the 2024 Royal International Air Tattoo show at RAF Fairford.\\\\\\\\\\\\n\\\\\\\\\\\\n4 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nEngland](/news/videos/cmj24x03r8po)\\\\n\\\\n[![Woman with glasses and nose ring](https://ichef.bbci.co.uk/news/480/cpsprodpb/757d/live/00b226b0-47b6-11ef-b74c-bb483a802c97.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nDemocrats in Michigan react to Biden dropping out\\\\\\\\\\\\n-------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nDemocratic voters in Michigan give their take on Joe Biden withdrawing from the US presidential race.\\\\\\\\\\\\n\\\\\\\\\\\\n18 hrs ago](/news/videos/c51yrr2z74no)\\\\n\\\\n[![Turkey\\'s answer to \\'Burning Man\\'](https://ichef.bbci.co.uk/images/ic/480x270/p0jchx33.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nTurkey\\'s answer to \\'Burning Man\\'\\\\\\\\\\\\n--------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nA music and art extravaganza takes place in an \\'otherworldly\\' landscape amid unique volcanic rock formations.\\\\\\\\\\\\n\\\\\\\\\\\\n13 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nCulture & Experiences](/reel/video/p0jch19q/turkey-s-answer-to-burning-man-)\\\\n\\\\n[![Jayson Tatum](https://ichef.bbci.co.uk/news/480/cpsprodpb/aac4/live/37ef8560-4837-11ef-b74c-bb483a802c97.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nTatum on handling criticism and \\'joy\\' of Olympics\\\\\\\\\\\\n-------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nTeam USA and Boston Celtics power forward Jayson Tatum says \\\\\"basketball can\\'t be your sole purpose\\\\\" as he speaks about facing criticism, and the \\\\\"joy\\\\\" that playing in the Olympics brings.\\\\\\\\\\\\n\\\\\\\\\\\\n2 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nOlympic Games](/sport/basketball/videos/cg640yk3n3zo)\\\\n\\\\n[![The world\\'s longest rowing boat](https://ichef.bbci.co.uk/news/480/cpsprodpb/598c/live/ee81bb50-4750-11ef-b74c-bb483a802c97.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nWorld\\'s longest rowing boat to carry Olympic torch\\\\\\\\\\\\n--------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nThe 24-seater boat will take the Olympic torch down a section of the River Marne on Sunday.\\\\\\\\\\\\n\\\\\\\\\\\\n1 day ago\\\\\\\\\\\\n\\\\\\\\\\\\nEurope](/news/videos/cqe6q917y1jo)\\\\n\\\\n[![Kellie Dingwall](https://ichef.bbci.co.uk/news/480/cpsprodpb/1d34/live/14fec1f0-4513-11ef-9e1c-3b4a473456a6.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nAccessibility brings disabled gamers a sense of community\\\\\\\\\\\\n---------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nGreater accessibility in game development is opening the genre to more people with disabilities.\\\\\\\\\\\\n\\\\\\\\\\\\n1 day ago\\\\\\\\\\\\n\\\\\\\\\\\\nScotland](/news/videos/c0jq593xqk8o)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![A plume of smoke rises above Dallas, Texas](https://ichef.bbci.co.uk/news/480/cpsprodpb/7bc6/live/13011710-46dd-11ef-9e1c-3b4a473456a6.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nWatch: Spire collapses as fire engulfs Texas church\\\\\\\\\\\\n---------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nA blaze at a historic church in Dallas has caused huge plumes of smoke to rise over the Texan city\\\\\\\\\\\\n\\\\\\\\\\\\n2 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nUS & Canada](/news/videos/cjk325014g7o)\\\\n\\\\n* * *\\\\n\\\\nIn History\\\\n----------\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![Stefan Westmann (Credit: BBC Archive)](credit: BBC Archive)\\\\\\\\\\\\n\\\\\\\\\\\\n\\'He was after my life\\': WW1 soldier\\'s confession\\\\\\\\\\\\n------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nWorld War One broke out on 28 July, 1914. Fifty years later, one of the German soldiers, Stefan Westmann, told the BBC about his experiences fighting in the conflict.\\\\\\\\\\\\n\\\\\\\\\\\\nSee more](/culture/article/20240718-a-ww1-soldier-on-the-brutality-of-conflict)\\\\n\\\\n* * *\\\\n\\\\n[Olympic Games\\\\\\\\\\\\n-------------](https://www.bbc.com/sport/olympics)\\\\n\\\\u00a0\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![Julian Alvarez and Marta](https://ichef.bbci.co.uk/news/480/cpsprodpb/76b3/live/1c0146e0-4756-11ef-967d-737978ba9b63.png.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nTen footballers to watch out for at Paris Olympics\\\\\\\\\\\\n--------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nFrom Manchester City\\'s Julian Alvarez to Brazil icon Marta, BBC Sport picks out 10 footballers to watch at the Olympics.\\\\\\\\\\\\n\\\\\\\\\\\\nSee more](/sport/football/articles/cek91m98g48o)\\\\n\\\\n* * *\\\\n\\\\nUS and Canada news\\\\n------------------\\\\n\\\\n[\\'The right move but is it too late?\\' Democratic voters react\\\\\\\\\\\\n------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nJust now\\\\\\\\\\\\n\\\\\\\\\\\\nUS & Canada](/news/articles/crgk8rgm87lo)\\\\n\\\\n[Who could be Kamala Harris\\'s running mate?\\\\\\\\\\\\n------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n10 mins ago\\\\\\\\\\\\n\\\\\\\\\\\\nUS & Canada](/news/articles/c80ekdwk9zro)\\\\n\\\\n[Biden has backed Harris. What happens next in US election?\\\\\\\\\\\\n----------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n18 mins ago\\\\\\\\\\\\n\\\\\\\\\\\\nUS & Canada](/news/articles/cq5xdq71drro)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![Kamala Harris speaking at the White House](https://ichef.bbci.co.uk/news/480/cpsprodpb/44e6/live/0560ada0-4845-11ef-b74c-bb483a802c97.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nBiden\\'s legacy of accomplishment is unmatched - Harris\\\\\\\\\\\\n------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nThe US Vice-President praised Joe Biden\\'s track record as US president.\\\\\\\\\\\\n\\\\\\\\\\\\n34 mins ago\\\\\\\\\\\\n\\\\\\\\\\\\nUS & Canada](/news/videos/c51y936y9g2o)\\\\n\\\\n* * *\\\\n\\\\nGlobal news\\\\n-----------\\\\n\\\\n[At least six killed in Croatia nursing home shooting\\\\\\\\\\\\n----------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n7 mins ago\\\\\\\\\\\\n\\\\\\\\\\\\nEurope](/news/articles/cn08d7vyj6wo)\\\\n\\\\n[Israel orders evacuation of part of Gaza humanitarian zone\\\\\\\\\\\\n----------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n2 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nMiddle East](/news/articles/cgerz8we1vgo)\\\\n\\\\n[Russian-US journalist jailed for \\'false information\\'\\\\\\\\\\\\n----------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n2 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nEurope](/news/articles/cn08d7j1qj5o)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![Former information minister of Tanzania, Nape Nnauye](https://ichef.bbci.co.uk/news/480/cpsprodpb/1f9f/live/ec833280-481c-11ef-96a8-e710c6bfc866.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nTanzanian minister sacked after poll rigging remarks\\\\\\\\\\\\n----------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nNape Nnauye caused outrage for saying he could help an MP rig elections - comments, he said, he made in jest.\\\\\\\\\\\\n\\\\\\\\\\\\n3 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nAfrica](/news/articles/cd1e48677w0o)\\\\n\\\\n* * *\\\\n\\\\nUK news\\\\n-------\\\\n\\\\n[Campaigners in court over Magna Carta incident\\\\\\\\\\\\n----------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n54 mins ago\\\\\\\\\\\\n\\\\\\\\\\\\nUK](/news/articles/cxw29lg4y8go)\\\\n\\\\n[New Prince George photo released on 11th birthday\\\\\\\\\\\\n-------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n7 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nUK](/news/articles/cw4yxd3dw7qo)\\\\n\\\\n[\\'We have one of the best comedy scenes in the UK\\'\\\\\\\\\\\\n-------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n12 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nUK](/news/articles/c4ngrdpwlx3o)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![Members of Parliament, newly elected in the 2024 general election, gather in the House of Commons Chamber for a group photo](https://ichef.bbci.co.uk/news/480/cpsprodpb/a921/live/f7aeabe0-4465-11ef-a959-6902b6903fc8.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nHow does a surprise MP prepare for life in office?\\\\\\\\\\\\n--------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nNew MPs in the East describe the experience of unexpectedly picking up the reins of public life.\\\\\\\\\\\\n\\\\\\\\\\\\n12 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nEngland](/news/articles/cxr2g7rk3dxo)\\\\n\\\\n* * *\\\\n\\\\nSport\\\\n-----\\\\n\\\\n[Ferdinand\\'s persuasive powers sealed Man Utd\\'s Yoro deal\\\\\\\\\\\\n--------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n49 mins ago\\\\\\\\\\\\n\\\\\\\\\\\\nMan Utd](/sport/football/articles/c9e950ev0xpo)\\\\n\\\\n[Aston Villa complete \\\\u00a350m deal for Everton\\'s Onana\\\\\\\\\\\\n--------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n2 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nPremier League](/sport/football/articles/cgrlgzx6gpro)\\\\n\\\\n[Australia would not pick convicted rapist Olympian\\\\\\\\\\\\n--------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n3 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nOlympic Games](/sport/articles/c2v0j0j6nqlo)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![Xander Schaufelle looks at the Claret Jug](https://ichef.bbci.co.uk/news/480/cpsprodpb/e404/live/2ccb8690-483f-11ef-96a8-e710c6bfc866.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\n\\'Schauffele passes ultimate examination in classic Open\\'\\\\\\\\\\\\n--------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nThe 152nd Open should be remembered as a classic, writes BBC golf correspondent Iain Carter.\\\\\\\\\\\\n\\\\\\\\\\\\n1 hr ago\\\\\\\\\\\\n\\\\\\\\\\\\nGolf](/sport/golf/articles/cv2g1glwypqo)\\\\n\\\\n* * *\\\\n\\\\n[Video\\\\\\\\\\\\n-----](https://www.bbc.com/video)\\\\n\\\\u00a0\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![Saving the real \\'Paddington Bear\\' in Bolivia](https://ichef.bbci.co.uk/images/ic/480x270/p0jbv23y.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nThe \\'Paddington bears\\' facing climate threat\\\\\\\\\\\\n--------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nDrought forces the real Paddington Bear\\\\u00a0into deadly conflict with cattle farmers in the Andes.\\\\\\\\\\\\n\\\\\\\\\\\\nSee more](/reel/video/p0jbv222/climate-chaos-makes-paddington-bear-hangry-)\\\\n\\\\n* * *\\\\n\\\\nBusiness\\\\n--------\\\\n\\\\n[Ryanair set to slash summer fares as profits drop\\\\\\\\\\\\n-------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n8 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nBusiness](/news/articles/cj50d6q3jlro)\\\\n\\\\n[Former chancellor Zahawi mulling bid for the Telegraph\\\\\\\\\\\\n------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n26 mins ago\\\\\\\\\\\\n\\\\\\\\\\\\nBusiness](/news/articles/c4ng5q4jd62o)\\\\n\\\\n[Prime sued in trademark case by US Olympic committee\\\\\\\\\\\\n----------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n1 day ago](/news/articles/c4ng785gjv0o)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![people at Melbourne airport](https://ichef.bbci.co.uk/news/480/cpsprodpb/2bda/live/f864fb00-46bf-11ef-93d9-870ecb62df8e.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nCrowdStrike IT outage affected 8.5 million Windows devices, Microsoft says\\\\\\\\\\\\n--------------------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nIt\\\\u2019s the first time that a number has been put on the glitch that is still causing problems around the world.\\\\\\\\\\\\n\\\\\\\\\\\\n2 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nTechnology](/news/articles/cpe3zgznwjno)\\\\n\\\\n* * *\\\\n\\\\nInnovation\\\\n----------\\\\n\\\\n[Company wins funding to make medicine in space\\\\\\\\\\\\n----------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n4 hrs ago](/news/articles/cp9vdxwjddeo)\\\\n\\\\n[Summer surge: why Covid-19 isn\\'t yet seasonal\\\\\\\\\\\\n---------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n1 day ago\\\\\\\\\\\\n\\\\\\\\\\\\nFuture](/future/article/20240719-why-covid-19-is-spreading-this-summer)\\\\n\\\\n[Scam warning as fake emails and websites target users after outage\\\\\\\\\\\\n------------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n2 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nTechnology](/news/articles/cq5xy12pynyo)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![A jar of kombucha against a pastel pink background (Credit: Getty Images)](credit: Getty Images)\\\\\\\\\\\\n\\\\\\\\\\\\nAre fermented foods actually good for our health?\\\\\\\\\\\\n-------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nWhile humans have been eating fermented foods since ancient times, researchers are only starting to unravel some of the biggest questions about their health benefits.\\\\\\\\\\\\n\\\\\\\\\\\\n2 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nFuture](/future/article/20240719-are-fermented-foods-actually-good-for-you)\\\\n\\\\n* * *\\\\n\\\\nCulture\\\\n-------\\\\n\\\\n[How to choose the most eco-friendly swimwear\\\\\\\\\\\\n--------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n2 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nCulture](/culture/article/20240719-how-to-choose-the-best-and-most-eco-friendly-swimwear)\\\\n\\\\n[In pictures: Colonial India through the eyes of foreign artists\\\\\\\\\\\\n---------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n2 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nAsia](/news/articles/c28ejl4nvgyo)\\\\n\\\\n[Auction for John Lennon glasses and Abbey Road photos\\\\\\\\\\\\n-----------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n1 day ago\\\\\\\\\\\\n\\\\\\\\\\\\nSurrey](/news/articles/cp085ym9l3ro)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![A picture of Bella Hadid](https://ichef.bbci.co.uk/news/480/cpsprodpb/da1b/live/4c41b850-4678-11ef-895c-59d6554481d1.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nBella Hadid\\'s Adidas advert dropped after Israeli criticism\\\\\\\\\\\\n-----------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nThe model was chosen to promote shoes referencing the 1972 Munich Olympics, at which 11 Israeli athletes were killed.\\\\\\\\\\\\n\\\\\\\\\\\\n2 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nCulture](/news/articles/ceqdwpv8vw1o)\\\\n\\\\n* * *\\\\n\\\\nTravel\\\\n------\\\\n\\\\n[The Indian dish Kamala Harris loves\\\\\\\\\\\\n-----------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n1 hr ago\\\\\\\\\\\\n\\\\\\\\\\\\nTravel](/travel/article/20201026-dosa-indias-wholesome-fast-food-obsession)\\\\n\\\\n[Italy\\'s most iconic trail reopens after 12 years\\\\\\\\\\\\n------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n3 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nTravel](/travel/article/20240722-cinque-terre-italys-path-of-love-reopens-after-12-years)\\\\n\\\\n[The US\\'s little-known \\'Ellis Island of the West\\'\\\\\\\\\\\\n------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n5 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nTravel](/travel/article/20240719-angel-island-the-little-known-ellis-island-of-the-west-us)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![Dondurma (Credit: Alamy)](credit: Alamy)\\\\\\\\\\\\n\\\\\\\\\\\\nThe Turkish ice cream eaten with a knife and fork\\\\\\\\\\\\n-------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nDondurma isn\\'t like any other ice cream you\\'ll find, and the epicentre of its production is still reeling from the powerful earthquakes that decimated the nation.\\\\\\\\\\\\n\\\\\\\\\\\\n1 day ago\\\\\\\\\\\\n\\\\\\\\\\\\nTravel](/travel/article/20240719-dondurma-the-turkish-ice-cream-eaten-with-a-knife-and-fork)\\\\n\\\\n* * *\\\\n\\\\n[Travel\\\\\\\\\\\\n------](https://www.bbc.com/travel)\\\\n\\\\u00a0\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![Angel Island (Credit: Alamy)](credit: Alamy)\\\\\\\\\\\\n\\\\\\\\\\\\nThe US\\'s little-known \\'Ellis Island of the West\\'\\\\\\\\\\\\n------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nThis former quarantine and military station once processed as many as one million immigrants. Now, the picturesque island is one of the San Francisco Bay Area\\'s best urban getaways.\\\\\\\\\\\\n\\\\\\\\\\\\nSee more](/travel/article/20240719-angel-island-the-little-known-ellis-island-of-the-west-us)\\\\n\\\\n* * *\\\\n\\\\nSign up for our newsletters\\\\n---------------------------\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![The Essential List](https://ichef.bbci.co.uk/images/ic/1920x1080/p0h74xp9.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nThe Essential List\\\\\\\\\\\\n------------------\\\\\\\\\\\\n\\\\\\\\\\\\nThe week\\'s best stories, handpicked by BBC editors, in your inbox every Tuesday and Friday.](https://cloud.email.bbc.com/SignUp10_08?&at_bbc_team=studios&at_medium=display&at_objective=acquisition&at_ptr_type=&at_ptr_name=bbc.comhp&at_format=Module&at_link_origin=homepage&at_campaign=essentiallist&at_campaign_type=owned)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![US Election Unspun](https://ichef.bbci.co.uk/images/ic/1920x1080/p0h74xqg.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nUS Election Unspun\\\\\\\\\\\\n------------------\\\\\\\\\\\\n\\\\\\\\\\\\nCut through the spin with North America correspondent Anthony Zurcher - in your inbox every Wednesday.](https://cloud.email.bbc.com/US_Election_Unspun_newsletter_signup?&at_bbc_team=studios&at_medium=display&at_objective=acquisition&at_ptr_type=&at_ptr_name=bbc.comhp&at_format=Module&at_link_origin=homepage&at_campaign=uselectionunspun&at_campaign_type=owned)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![Medal Moments](https://ichef.bbci.co.uk/images/ic/raw/p0j53vjd.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nMedal Moments\\\\\\\\\\\\n-------------\\\\\\\\\\\\n\\\\\\\\\\\\nYour global guide to the Paris Olympics, from key highlights to heroic stories, daily to your inbox throughout the Games.](https://cloud.email.bbc.com/medalmoments_newsletter_signup?&at_bbc_team=studios&at_medium=emails&at_objective=acquisition&at_ptr_type=&at_ptr_name=bbc.com&at_format=Module&at_link_origin=homepage&at_campaign=olympics2024&at_campaign_id=&at_adset_name=&at_adset_id=&at_creation=&at_creative_id=&at_campaign_type=owned)\\\\n\\\\n* * *\\\\n\\\\nEarth\\\\n-----\\\\n\\\\n[The \\'Paddington bears\\' facing climate threat\\\\\\\\\\\\n--------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n2 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nNatural wonders](/reel/video/p0jbv222/climate-chaos-makes-paddington-bear-hangry-)\\\\n\\\\n[The simple Japanese method for a tidier fridge\\\\\\\\\\\\n----------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n3 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nFuture](/future/article/20240715-the-simple-japanese-method-for-a-tidier-and-less-wasteful-fridge)\\\\n\\\\n[Conspiracy theories swirl about geo-engineering, but could it help save the planet?\\\\\\\\\\\\n-----------------------------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n2 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nBBC InDepth](/news/articles/c98qp79gj4no)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![Humpback whale](https://ichef.bbci.co.uk/images/ic/480x270/p0jbqk2h.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nFace to face with humpback whales\\\\\\\\\\\\n---------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nReece Parkinson discovers how locals are protecting their stunning marine environment.\\\\\\\\\\\\n\\\\\\\\\\\\n3 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nTravel](/reel/video/p0jbpt60/face-to-face-with-humpback-whales)\\\\n\\\\n* * *\\\\n\\\\nScience and health\\\\n------------------\\\\n\\\\n[India alert after boy dies from Nipah virus in Kerala\\\\\\\\\\\\n-----------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n6 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nAsia](/news/articles/cj50d7e9vp6o)\\\\n\\\\n[Are fermented foods actually good for our health?\\\\\\\\\\\\n-------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n2 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nFuture](/future/article/20240719-are-fermented-foods-actually-good-for-you)\\\\n\\\\n[Summer surge: why Covid-19 isn\\'t yet seasonal\\\\\\\\\\\\n---------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n1 day ago\\\\\\\\\\\\n\\\\\\\\\\\\nFuture](/future/article/20240719-why-covid-19-is-spreading-this-summer)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![A young woman\\'s face with a red background](https://ichef.bbci.co.uk/news/1536/cpsprodpb/3884/live/affa3d80-45e3-11ef-837a-9936fb8608b6.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\n\\'A way to fight back\\': FGM survivors reclaim vulvas with surgery\\\\\\\\\\\\n----------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nIn Somalia, it\\'s believed cutting off a girl\\'s outer genitalia will guarantee their virginity.\\\\\\\\\\\\n\\\\\\\\\\\\n2 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nAfrica](/news/articles/cyx0perl8yno)\\\\n\\\\n* * *\\\\n\\\\nWorld\\\\u2019s Table\\\\n-------------\\\\n\\\\n[The Turkish ice cream eaten with a knife and fork\\\\\\\\\\\\n-------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n1 day ago\\\\\\\\\\\\n\\\\\\\\\\\\nTravel](/travel/article/20240719-dondurma-the-turkish-ice-cream-eaten-with-a-knife-and-fork)\\\\n\\\\n[India\\'s cooling drinks to beat the heat\\\\\\\\\\\\n---------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n7 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nTravel](/travel/article/20240715-gond-katira-a-natural-way-to-cool-down-in-indias-scorching-summers)\\\\n\\\\n[The world\\'s biggest restaurant is coming to Paris\\\\\\\\\\\\n-------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n9 Jul 2024\\\\\\\\\\\\n\\\\\\\\\\\\nTravel](/travel/article/20240708-paris-olympics-2024-worlds-biggest-restaurant)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![Plate of \\\\u00e7i\\\\u011f k\\\\u00f6fte made with raw meat (Credit: Paul Benjamin Osterlund)](credit: Paul Benjamin Osterlund)\\\\\\\\\\\\n\\\\\\\\\\\\nThe wild ceremonies of the Turkish \\'meatball\\'\\\\\\\\\\\\n---------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nOne of the country\\'s most popular fast-food items, \\\\u00e7i\\\\u011f k\\\\u00f6fte is traditionally associated with wild and rowdy gatherings in south-eastern Turkey.\\\\\\\\\\\\n\\\\\\\\\\\\n1 Jun 2024\\\\\\\\\\\\n\\\\\\\\\\\\nTravel](/travel/article/20240531-the-wild-ceremonies-surrounding-a-turkish-meatball)\\\\n\\\\n* * *\\\\n\\\\nThe Specialist\\\\n--------------\\\\n\\\\n[Guide to Helsinki\\'s happiest places\\\\\\\\\\\\n-----------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n2 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nTravel](/travel/article/20240718-a-happiness-hackers-guide-to-the-happiest-outdoor-places-in-helsinki)\\\\n\\\\n[A pastry chef\\'s favourite bakeries in Paris\\\\\\\\\\\\n-------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n5 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nTravel](/travel/article/20240717-david-lebovitzs-ultimate-guide-to-the-best-bakeries-in-paris-right-now)\\\\n\\\\n[Chef Andrew Zimmern\\'s favourite US restaurants\\\\\\\\\\\\n----------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n14 Jul 2024\\\\\\\\\\\\n\\\\\\\\\\\\nTravel](/travel/article/20240714-chef-andrew-zimmerns-favourite-us-restaurants)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![Iceland in winter (Credit: Getty Images)](credit: Getty Images)\\\\\\\\\\\\n\\\\\\\\\\\\nA First Lady\\'s guide to Iceland\\\\\\\\\\\\n-------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nEliza Reid moved to Iceland 20 years ago for love and now she\\'s the First Lady. Here are her favourite ways to enjoy a \\\\\"chill\\\\\" Icelandic weekend.\\\\\\\\\\\\n\\\\\\\\\\\\n11 Jul 2024\\\\\\\\\\\\n\\\\\\\\\\\\nTravel](/travel/article/20240710-icelands-first-lady-takes-you-on-a-tour-of-her-super-chill-nation)\\\\n\\\\n* * *\\\\n\\\\n[British Broadcasting Corporation](/)\\\\n\\\\n* [Home](https://www.bbc.com/)\\\\n \\\\n* [News](/news)\\\\n \\\\n* [Sport](/sport)\\\\n \\\\n* [Business](/business)\\\\n \\\\n* [Innovation](/innovation)\\\\n \\\\n* [Culture](/culture)\\\\n \\\\n* [Travel](/travel)\\\\n \\\\n* [Earth](/future-planet)\\\\n \\\\n* [Video](/video)\\\\n \\\\n* [Live](/live)\\\\n \\\\n* [Audio](https://www.bbc.co.uk/sounds)\\\\n \\\\n* [Weather](https://www.bbc.com/weather)\\\\n \\\\n* [BBC Shop](https://shop.bbc.com/)\\\\n \\\\n\\\\nBBC in other languages\\\\n\\\\nFollow BBC on:\\\\n--------------\\\\n\\\\n* [Terms of Use](https://www.bbc.co.uk/usingthebbc/terms)\\\\n \\\\n* [About the BBC](https://www.bbc.co.uk/aboutthebbc)\\\\n \\\\n* [Privacy Policy](https://www.bbc.com/usingthebbc/privacy/)\\\\n \\\\n* [Cookies](https://www.bbc.com/usingthebbc/cookies/)\\\\n \\\\n* [Accessibility Help](https://www.bbc.co.uk/accessibility/)\\\\n \\\\n* [Contact the BBC](https://www.bbc.co.uk/contact)\\\\n \\\\n* [Advertise with us](https://www.bbc.com/advertisingcontact)\\\\n \\\\n* [Do not share or sell my info](https://www.bbc.com/usingthebbc/cookies/how-can-i-change-my-bbc-cookie-settings/)\\\\n \\\\n* [Contact technical support](https://www.bbc.com/contact-bbc-com-help)\\\\n \\\\n\\\\nCopyright 2024 BBC. All rights reserved.\\\\u00a0\\\\u00a0The _BBC_ is _not responsible for the content of external sites._\\\\u00a0[**Read about our approach to external linking.**](https://www.bbc.co.uk/editorialguidelines/guidance/feeds-and-links)\", \"markdown\": \"\\\\n\\\\n[British Broadcasting Corporation](/)\\\\n\\\\n[Watch Live](/watch-live-news)\\\\n\\\\nRegisterSign In\\\\n\\\\n* [Home](/)\\\\n \\\\n* [News](/news)\\\\n \\\\n* [Sport](/sport)\\\\n \\\\n* [Business](/business)\\\\n \\\\n* [Innovation](/innovation)\\\\n \\\\n* [Culture](/culture)\\\\n \\\\n* [Travel](/travel)\\\\n \\\\n* [Earth](/future-planet)\\\\n \\\\n* [Video](/video)\\\\n \\\\n* [Live](/live)\\\\n \\\\n\\\\nRegisterSign In\\\\n\\\\n[Home](/)\\\\n\\\\nNews\\\\n\\\\n[Sport](/sport)\\\\n\\\\nBusiness\\\\n\\\\nInnovation\\\\n\\\\nCulture\\\\n\\\\nTravel\\\\n\\\\nEarth\\\\n\\\\n[Video](/video)\\\\n\\\\nLive\\\\n\\\\n[Audio](https://www.bbc.co.uk/sounds)\\\\n\\\\n[Weather](https://www.bbc.com/weather)\\\\n\\\\n[Newsletters](https://www.bbc.com/newsletters)\\\\n\\\\n[![Biden\\'s last public appearance in Delaware before announcing he would withdraw from the race](https://ichef.bbci.co.uk/news/480/cpsprodpb/05a8/live/0dce30e0-47ee-11ef-96a8-e710c6bfc866.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nThe last days of the Biden campaign \\\\u2013 BBC correspondent\\\\u2019s account\\\\\\\\\\\\n-----------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nBBC correspondent Tom Bateman takes us behind the scenes in the final days of Biden\\'s campaign.\\\\\\\\\\\\n\\\\\\\\\\\\n3 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nUS & Canada](/news/articles/c250zqgrpqgo)\\\\n\\\\n[![Joe Biden looks on during a speech at the White House on 14 July ](https://ichef.bbci.co.uk/news/480/cpsprodpb/ffdd/live/f2086770-4808-11ef-96a8-e710c6bfc866.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nIsolating at a beach house, Biden gave aides one minute notice of exit\\\\\\\\\\\\n----------------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nMost of the president\\'s senior aides found out about his decision to exit the race minutes before he publicly announced it.\\\\\\\\\\\\n\\\\\\\\\\\\n15 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nUS & Canada](/news/articles/c3gdzmdje5xo)\\\\n\\\\n[![Biden\\'s last public appearance in Delaware before announcing he would withdraw from the race](https://ichef.bbci.co.uk/news/480/cpsprodpb/05a8/live/0dce30e0-47ee-11ef-96a8-e710c6bfc866.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nThe last days of the Biden campaign \\\\u2013 BBC correspondent\\\\u2019s account\\\\\\\\\\\\n-----------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nBBC correspondent Tom Bateman takes us behind the scenes in the final days of Biden\\'s campaign.\\\\\\\\\\\\n\\\\\\\\\\\\n3 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nUS & Canada](/news/articles/c250zqgrpqgo)\\\\n\\\\n[![Joe Biden looks on during a speech at the White House on 14 July ](https://ichef.bbci.co.uk/news/480/cpsprodpb/ffdd/live/f2086770-4808-11ef-96a8-e710c6bfc866.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nIsolating at a beach house, Biden gave aides one minute notice of exit\\\\\\\\\\\\n----------------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nMost of the president\\'s senior aides found out about his decision to exit the race minutes before he publicly announced it.\\\\\\\\\\\\n\\\\\\\\\\\\n15 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nUS & Canada](/news/articles/c3gdzmdje5xo)\\\\n\\\\n[![Kamala Harris gestures as she speaks at the White House on Monday](https://ichef.bbci.co.uk/ace/standard/480/cpsprodpb/7856/live/1ce684a0-4844-11ef-9e1c-3b4a473456a6.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nLIVE\\\\\\\\\\\\n\\\\\\\\\\\\nKamala Harris speaks for first time since Biden left race - as endorsements mount\\\\\\\\\\\\n---------------------------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nThe US vice-president appears at the White House for a pre-scheduled event - as key Democrats line up to back her candidacy.](https://www.bbc.com/news/live/cv2gryx1yx1t)\\\\n\\\\n* * *\\\\n\\\\n[What Biden quitting means for Harris, the Democrats and Trump\\\\\\\\\\\\n-------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nPresident Biden has upended the 2024 White House race for the Democrats. Here is what it means for Kamala Harris, his party and Trump.\\\\\\\\\\\\n\\\\\\\\\\\\n19 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nUS & Canada](/news/articles/cpwd8yzw45qo)\\\\n\\\\n[Who could be Kamala Harris\\'s running mate?\\\\\\\\\\\\n------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nIt\\\\u2019s not a done deal but some potential rivals have quickly thrown their support behind her.\\\\\\\\\\\\n\\\\\\\\\\\\n10 mins ago\\\\\\\\\\\\n\\\\\\\\\\\\nUS & Canada](/news/articles/c80ekdwk9zro)\\\\n\\\\n[LIVE\\\\\\\\\\\\n\\\\\\\\\\\\nTensions flare as Congress presses Secret Service boss on \\'failed\\' Trump rally security\\\\\\\\\\\\n---------------------------------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nMembers of both parties have called for Kimberly Cheatle to resign in a House committee hearing that is seeking answers over security failures at the rally on 13 July.](https://www.bbc.com/news/live/c724wqpy4qnt)\\\\n\\\\n[The president\\'s protectors are hardly noticeable - until things go wrong\\\\\\\\\\\\n------------------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nThe attempted assassination of Donald Trump has raised questions about the Secret Service\\'s record.\\\\\\\\\\\\n\\\\\\\\\\\\n6 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nUS & Canada](/news/articles/c16j896003xo)\\\\n\\\\n* * *\\\\n\\\\nOnly from the BBC\\\\n-----------------\\\\n\\\\n[![Two people sit in deck chairs surrounded by water (Credit: Getty Images)](credit: Getty Images)\\\\\\\\\\\\n\\\\\\\\\\\\nWhy you are probably sitting for too long\\\\\\\\\\\\n-----------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nSitting down is ingrained in most peoples\\' days. But staying sedentary for too long can increase the risk of serious health conditions like cardiovascular disease and type 2 diabetes.\\\\\\\\\\\\n\\\\\\\\\\\\n4 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nFuture](/future/article/20240722-why-you-are-probably-sitting-down-for-too-long)\\\\n\\\\n[![South African convicted killer Louis van Schoor stares into the middle distance](https://ichef.bbci.co.uk/news/480/cpsprodpb/229c/live/a8aaa380-4520-11ef-b74c-bb483a802c97.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nMass killer who \\\\u2018hunted\\\\u2019 black people says police encouraged him\\\\\\\\\\\\n----------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nEx-security guard, Louis van Schoor, killed dozens in South Africa but was only jailed for seven murders.\\\\\\\\\\\\n\\\\\\\\\\\\n17 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nAfrica](/news/articles/c51yqdy3q61o)\\\\n\\\\n* * *\\\\n\\\\n[More news\\\\\\\\\\\\n---------](https://www.bbc.com/news)\\\\n\\\\u00a0\\\\n\\\\n[![File photo showing downtown Dubai\\'s skyline (12 June 2021)](https://www.bbc.com/12%20June%202021)\\\\\\\\\\\\n\\\\\\\\\\\\nUAE jails 57 Bangladeshis over protests against own government\\\\\\\\\\\\n--------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nThree defendants were sentenced to life after being convicted of \\\\\"inciting riots\\\\\" in the Gulf state.\\\\\\\\\\\\n\\\\\\\\\\\\n5 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nWorld](/news/articles/crgk8gnpg0zo)\\\\n\\\\n[![A Palestinian woman sitting on a wheelchair is pulled by a man as they flee eastern Khan Younis in response to an Israeli evacuation order, in the southern Gaza Strip (22 July 2024)](https://www.bbc.com/22%20July%202024)\\\\\\\\\\\\n\\\\\\\\\\\\nIsrael orders evacuation of part of Gaza humanitarian zone\\\\\\\\\\\\n----------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nStrikes are reported near Khan Younis, after people are told to leave eastern areas in the zone.\\\\\\\\\\\\n\\\\\\\\\\\\n2 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nMiddle East](/news/articles/cgerz8we1vgo)\\\\n\\\\n[![Black and white photo of Prince George in shirt and suit](https://ichef.bbci.co.uk/news/480/cpsprodpb/d483/live/f8c3d070-480a-11ef-9e1c-3b4a473456a6.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nNew Prince George photo released on 11th birthday\\\\\\\\\\\\n-------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nKensington Palace has posted the image, taken by his mother, Catherine, Princess of Wales.\\\\\\\\\\\\n\\\\\\\\\\\\n7 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nUK](/news/articles/cw4yxd3dw7qo)\\\\n\\\\n[Piastri wins in Hungary after Norris team orders row\\\\\\\\\\\\n----------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nOscar Piastri takes his maiden grand prix victory ahead of McLaren team-mate Lando Norris in a dramatic race in Hungary.\\\\\\\\\\\\n\\\\\\\\\\\\n1 day ago\\\\\\\\\\\\n\\\\\\\\\\\\nFormula 1](/sport/formula1/articles/cg3jzy3e8q1o)\\\\n\\\\n[India alert after boy dies from Nipah virus in Kerala\\\\\\\\\\\\n-----------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nThe virus is transmitted from animals such as pigs and fruit bats to humans.\\\\\\\\\\\\n\\\\\\\\\\\\n6 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nAsia](/news/articles/cj50d7e9vp6o)\\\\n\\\\n[![A Palestinian woman sitting on a wheelchair is pulled by a man as they flee eastern Khan Younis in response to an Israeli evacuation order, in the southern Gaza Strip (22 July 2024)](https://www.bbc.com/22%20July%202024)\\\\\\\\\\\\n\\\\\\\\\\\\nIsrael orders evacuation of part of Gaza humanitarian zone\\\\\\\\\\\\n----------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nStrikes are reported near Khan Younis, after people are told to leave eastern areas in the zone.\\\\\\\\\\\\n\\\\\\\\\\\\n2 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nMiddle East](/news/articles/cgerz8we1vgo)\\\\n\\\\n[![A Kanwariya carries holy water collected from Ganga River in Haridwar during Kanwar Yatra at Sector 14A, on July 10, 2023 in Noida](https://ichef.bbci.co.uk/news/480/cpsprodpb/b9f0/live/7390b470-47e9-11ef-80f2-8f08f27244b7.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nIndia court blocks order for eateries to display owners\\' names\\\\\\\\\\\\n--------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nCritics say ordering restaurants to prominently display names of owners is discriminatory towards Muslims.\\\\\\\\\\\\n\\\\\\\\\\\\n6 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nAsia](/news/articles/czrj18yp489o)\\\\n\\\\n[![Information screen informs train travellers of global IT outage. ](https://ichef.bbci.co.uk/news/480/cpsprodpb/bb5b/live/363718b0-47c3-11ef-be99-e9774e2b4d6b.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\n\\'Significant number\\' of devices fixed - CrowdStrike\\\\\\\\\\\\n---------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nCybersecurity firm behind global outage says it continues to focus on restoring all impacted computers.\\\\\\\\\\\\n\\\\\\\\\\\\n13 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nBusiness](/news/articles/cgl7e33n1d0o)\\\\n\\\\n[![An employee uses a jigsaw to cut a plastic pipe at the Grundfos AS factory in Chennai, India, on Monday, Nov. 27, 2017.](https://ichef.bbci.co.uk/news/480/cpsprodpb/7f64/live/70d23da0-45ad-11ef-b74c-bb483a802c97.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nModi\\'s new budget faces jobs crisis test in India\\\\\\\\\\\\n-------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nMr Modi\\'s biggest challenge in his third term will be bridging the rich-poor divide.\\\\\\\\\\\\n\\\\\\\\\\\\n16 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nAsia](/news/articles/cq5jwyel12qo)\\\\n\\\\n[![Black and white photo of Prince George in shirt and suit](https://ichef.bbci.co.uk/news/480/cpsprodpb/d483/live/f8c3d070-480a-11ef-9e1c-3b4a473456a6.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nNew Prince George photo released on 11th birthday\\\\\\\\\\\\n-------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nKensington Palace has posted the image, taken by his mother, Catherine, Princess of Wales.\\\\\\\\\\\\n\\\\\\\\\\\\n7 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nUK](/news/articles/cw4yxd3dw7qo)\\\\n\\\\n* * *\\\\n\\\\nMust watch\\\\n----------\\\\n\\\\n[![Biden speaks at a rally](https://ichef.bbci.co.uk/news/480/cpsprodpb/cc3c/live/cd687d10-4808-11ef-b74c-bb483a802c97.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nBiden\\\\u2019s disastrous few weeks... in 90 seconds\\\\\\\\\\\\n---------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nPresident Biden has faced intense pressure to step aside since his faltering debate performance.\\\\\\\\\\\\n\\\\\\\\\\\\n17 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nUS & Canada](/news/videos/cx028eq4qg1o)\\\\n\\\\n[![Royal Netherlands Navy\\'s anti-submarine helicopter](https://ichef.bbci.co.uk/news/480/cpsprodpb/80df/live/5d6dff00-482e-11ef-9e1c-3b4a473456a6.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nInside the Netherlands Navy\\'s anti-submarine helicopter\\\\\\\\\\\\n-------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nThe NH90 was on display at the 2024 Royal International Air Tattoo show at RAF Fairford.\\\\\\\\\\\\n\\\\\\\\\\\\n4 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nEngland](/news/videos/cmj24x03r8po)\\\\n\\\\n[![Woman with glasses and nose ring](https://ichef.bbci.co.uk/news/480/cpsprodpb/757d/live/00b226b0-47b6-11ef-b74c-bb483a802c97.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nDemocrats in Michigan react to Biden dropping out\\\\\\\\\\\\n-------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nDemocratic voters in Michigan give their take on Joe Biden withdrawing from the US presidential race.\\\\\\\\\\\\n\\\\\\\\\\\\n18 hrs ago](/news/videos/c51yrr2z74no)\\\\n\\\\n[![Turkey\\'s answer to \\'Burning Man\\'](https://ichef.bbci.co.uk/images/ic/480x270/p0jchx33.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nTurkey\\'s answer to \\'Burning Man\\'\\\\\\\\\\\\n--------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nA music and art extravaganza takes place in an \\'otherworldly\\' landscape amid unique volcanic rock formations.\\\\\\\\\\\\n\\\\\\\\\\\\n13 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nCulture & Experiences](/reel/video/p0jch19q/turkey-s-answer-to-burning-man-)\\\\n\\\\n[![Jayson Tatum](https://ichef.bbci.co.uk/news/480/cpsprodpb/aac4/live/37ef8560-4837-11ef-b74c-bb483a802c97.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nTatum on handling criticism and \\'joy\\' of Olympics\\\\\\\\\\\\n-------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nTeam USA and Boston Celtics power forward Jayson Tatum says \\\\\"basketball can\\'t be your sole purpose\\\\\" as he speaks about facing criticism, and the \\\\\"joy\\\\\" that playing in the Olympics brings.\\\\\\\\\\\\n\\\\\\\\\\\\n2 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nOlympic Games](/sport/basketball/videos/cg640yk3n3zo)\\\\n\\\\n[![The world\\'s longest rowing boat](https://ichef.bbci.co.uk/news/480/cpsprodpb/598c/live/ee81bb50-4750-11ef-b74c-bb483a802c97.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nWorld\\'s longest rowing boat to carry Olympic torch\\\\\\\\\\\\n--------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nThe 24-seater boat will take the Olympic torch down a section of the River Marne on Sunday.\\\\\\\\\\\\n\\\\\\\\\\\\n1 day ago\\\\\\\\\\\\n\\\\\\\\\\\\nEurope](/news/videos/cqe6q917y1jo)\\\\n\\\\n[![Kellie Dingwall](https://ichef.bbci.co.uk/news/480/cpsprodpb/1d34/live/14fec1f0-4513-11ef-9e1c-3b4a473456a6.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nAccessibility brings disabled gamers a sense of community\\\\\\\\\\\\n---------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nGreater accessibility in game development is opening the genre to more people with disabilities.\\\\\\\\\\\\n\\\\\\\\\\\\n1 day ago\\\\\\\\\\\\n\\\\\\\\\\\\nScotland](/news/videos/c0jq593xqk8o)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![A plume of smoke rises above Dallas, Texas](https://ichef.bbci.co.uk/news/480/cpsprodpb/7bc6/live/13011710-46dd-11ef-9e1c-3b4a473456a6.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nWatch: Spire collapses as fire engulfs Texas church\\\\\\\\\\\\n---------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nA blaze at a historic church in Dallas has caused huge plumes of smoke to rise over the Texan city\\\\\\\\\\\\n\\\\\\\\\\\\n2 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nUS & Canada](/news/videos/cjk325014g7o)\\\\n\\\\n* * *\\\\n\\\\nIn History\\\\n----------\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![Stefan Westmann (Credit: BBC Archive)](credit: BBC Archive)\\\\\\\\\\\\n\\\\\\\\\\\\n\\'He was after my life\\': WW1 soldier\\'s confession\\\\\\\\\\\\n------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nWorld War One broke out on 28 July, 1914. Fifty years later, one of the German soldiers, Stefan Westmann, told the BBC about his experiences fighting in the conflict.\\\\\\\\\\\\n\\\\\\\\\\\\nSee more](/culture/article/20240718-a-ww1-soldier-on-the-brutality-of-conflict)\\\\n\\\\n* * *\\\\n\\\\n[Olympic Games\\\\\\\\\\\\n-------------](https://www.bbc.com/sport/olympics)\\\\n\\\\u00a0\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![Julian Alvarez and Marta](https://ichef.bbci.co.uk/news/480/cpsprodpb/76b3/live/1c0146e0-4756-11ef-967d-737978ba9b63.png.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nTen footballers to watch out for at Paris Olympics\\\\\\\\\\\\n--------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nFrom Manchester City\\'s Julian Alvarez to Brazil icon Marta, BBC Sport picks out 10 footballers to watch at the Olympics.\\\\\\\\\\\\n\\\\\\\\\\\\nSee more](/sport/football/articles/cek91m98g48o)\\\\n\\\\n* * *\\\\n\\\\nUS and Canada news\\\\n------------------\\\\n\\\\n[\\'The right move but is it too late?\\' Democratic voters react\\\\\\\\\\\\n------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nJust now\\\\\\\\\\\\n\\\\\\\\\\\\nUS & Canada](/news/articles/crgk8rgm87lo)\\\\n\\\\n[Who could be Kamala Harris\\'s running mate?\\\\\\\\\\\\n------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n10 mins ago\\\\\\\\\\\\n\\\\\\\\\\\\nUS & Canada](/news/articles/c80ekdwk9zro)\\\\n\\\\n[Biden has backed Harris. What happens next in US election?\\\\\\\\\\\\n----------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n18 mins ago\\\\\\\\\\\\n\\\\\\\\\\\\nUS & Canada](/news/articles/cq5xdq71drro)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![Kamala Harris speaking at the White House](https://ichef.bbci.co.uk/news/480/cpsprodpb/44e6/live/0560ada0-4845-11ef-b74c-bb483a802c97.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nBiden\\'s legacy of accomplishment is unmatched - Harris\\\\\\\\\\\\n------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nThe US Vice-President praised Joe Biden\\'s track record as US president.\\\\\\\\\\\\n\\\\\\\\\\\\n34 mins ago\\\\\\\\\\\\n\\\\\\\\\\\\nUS & Canada](/news/videos/c51y936y9g2o)\\\\n\\\\n* * *\\\\n\\\\nGlobal news\\\\n-----------\\\\n\\\\n[At least six killed in Croatia nursing home shooting\\\\\\\\\\\\n----------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n7 mins ago\\\\\\\\\\\\n\\\\\\\\\\\\nEurope](/news/articles/cn08d7vyj6wo)\\\\n\\\\n[Israel orders evacuation of part of Gaza humanitarian zone\\\\\\\\\\\\n----------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n2 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nMiddle East](/news/articles/cgerz8we1vgo)\\\\n\\\\n[Russian-US journalist jailed for \\'false information\\'\\\\\\\\\\\\n----------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n2 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nEurope](/news/articles/cn08d7j1qj5o)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![Former information minister of Tanzania, Nape Nnauye](https://ichef.bbci.co.uk/news/480/cpsprodpb/1f9f/live/ec833280-481c-11ef-96a8-e710c6bfc866.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nTanzanian minister sacked after poll rigging remarks\\\\\\\\\\\\n----------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nNape Nnauye caused outrage for saying he could help an MP rig elections - comments, he said, he made in jest.\\\\\\\\\\\\n\\\\\\\\\\\\n3 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nAfrica](/news/articles/cd1e48677w0o)\\\\n\\\\n* * *\\\\n\\\\nUK news\\\\n-------\\\\n\\\\n[Campaigners in court over Magna Carta incident\\\\\\\\\\\\n----------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n54 mins ago\\\\\\\\\\\\n\\\\\\\\\\\\nUK](/news/articles/cxw29lg4y8go)\\\\n\\\\n[New Prince George photo released on 11th birthday\\\\\\\\\\\\n-------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n7 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nUK](/news/articles/cw4yxd3dw7qo)\\\\n\\\\n[\\'We have one of the best comedy scenes in the UK\\'\\\\\\\\\\\\n-------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n12 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nUK](/news/articles/c4ngrdpwlx3o)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![Members of Parliament, newly elected in the 2024 general election, gather in the House of Commons Chamber for a group photo](https://ichef.bbci.co.uk/news/480/cpsprodpb/a921/live/f7aeabe0-4465-11ef-a959-6902b6903fc8.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nHow does a surprise MP prepare for life in office?\\\\\\\\\\\\n--------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nNew MPs in the East describe the experience of unexpectedly picking up the reins of public life.\\\\\\\\\\\\n\\\\\\\\\\\\n12 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nEngland](/news/articles/cxr2g7rk3dxo)\\\\n\\\\n* * *\\\\n\\\\nSport\\\\n-----\\\\n\\\\n[Ferdinand\\'s persuasive powers sealed Man Utd\\'s Yoro deal\\\\\\\\\\\\n--------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n49 mins ago\\\\\\\\\\\\n\\\\\\\\\\\\nMan Utd](/sport/football/articles/c9e950ev0xpo)\\\\n\\\\n[Aston Villa complete \\\\u00a350m deal for Everton\\'s Onana\\\\\\\\\\\\n--------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n2 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nPremier League](/sport/football/articles/cgrlgzx6gpro)\\\\n\\\\n[Australia would not pick convicted rapist Olympian\\\\\\\\\\\\n--------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n3 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nOlympic Games](/sport/articles/c2v0j0j6nqlo)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![Xander Schaufelle looks at the Claret Jug](https://ichef.bbci.co.uk/news/480/cpsprodpb/e404/live/2ccb8690-483f-11ef-96a8-e710c6bfc866.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\n\\'Schauffele passes ultimate examination in classic Open\\'\\\\\\\\\\\\n--------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nThe 152nd Open should be remembered as a classic, writes BBC golf correspondent Iain Carter.\\\\\\\\\\\\n\\\\\\\\\\\\n1 hr ago\\\\\\\\\\\\n\\\\\\\\\\\\nGolf](/sport/golf/articles/cv2g1glwypqo)\\\\n\\\\n* * *\\\\n\\\\n[Video\\\\\\\\\\\\n-----](https://www.bbc.com/video)\\\\n\\\\u00a0\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![Saving the real \\'Paddington Bear\\' in Bolivia](https://ichef.bbci.co.uk/images/ic/480x270/p0jbv23y.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nThe \\'Paddington bears\\' facing climate threat\\\\\\\\\\\\n--------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nDrought forces the real Paddington Bear\\\\u00a0into deadly conflict with cattle farmers in the Andes.\\\\\\\\\\\\n\\\\\\\\\\\\nSee more](/reel/video/p0jbv222/climate-chaos-makes-paddington-bear-hangry-)\\\\n\\\\n* * *\\\\n\\\\nBusiness\\\\n--------\\\\n\\\\n[Ryanair set to slash summer fares as profits drop\\\\\\\\\\\\n-------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n8 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nBusiness](/news/articles/cj50d6q3jlro)\\\\n\\\\n[Former chancellor Zahawi mulling bid for the Telegraph\\\\\\\\\\\\n------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n26 mins ago\\\\\\\\\\\\n\\\\\\\\\\\\nBusiness](/news/articles/c4ng5q4jd62o)\\\\n\\\\n[Prime sued in trademark case by US Olympic committee\\\\\\\\\\\\n----------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n1 day ago](/news/articles/c4ng785gjv0o)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![people at Melbourne airport](https://ichef.bbci.co.uk/news/480/cpsprodpb/2bda/live/f864fb00-46bf-11ef-93d9-870ecb62df8e.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nCrowdStrike IT outage affected 8.5 million Windows devices, Microsoft says\\\\\\\\\\\\n--------------------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nIt\\\\u2019s the first time that a number has been put on the glitch that is still causing problems around the world.\\\\\\\\\\\\n\\\\\\\\\\\\n2 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nTechnology](/news/articles/cpe3zgznwjno)\\\\n\\\\n* * *\\\\n\\\\nInnovation\\\\n----------\\\\n\\\\n[Company wins funding to make medicine in space\\\\\\\\\\\\n----------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n4 hrs ago](/news/articles/cp9vdxwjddeo)\\\\n\\\\n[Summer surge: why Covid-19 isn\\'t yet seasonal\\\\\\\\\\\\n---------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n1 day ago\\\\\\\\\\\\n\\\\\\\\\\\\nFuture](/future/article/20240719-why-covid-19-is-spreading-this-summer)\\\\n\\\\n[Scam warning as fake emails and websites target users after outage\\\\\\\\\\\\n------------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n2 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nTechnology](/news/articles/cq5xy12pynyo)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![A jar of kombucha against a pastel pink background (Credit: Getty Images)](credit: Getty Images)\\\\\\\\\\\\n\\\\\\\\\\\\nAre fermented foods actually good for our health?\\\\\\\\\\\\n-------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nWhile humans have been eating fermented foods since ancient times, researchers are only starting to unravel some of the biggest questions about their health benefits.\\\\\\\\\\\\n\\\\\\\\\\\\n2 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nFuture](/future/article/20240719-are-fermented-foods-actually-good-for-you)\\\\n\\\\n* * *\\\\n\\\\nCulture\\\\n-------\\\\n\\\\n[How to choose the most eco-friendly swimwear\\\\\\\\\\\\n--------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n2 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nCulture](/culture/article/20240719-how-to-choose-the-best-and-most-eco-friendly-swimwear)\\\\n\\\\n[In pictures: Colonial India through the eyes of foreign artists\\\\\\\\\\\\n---------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n2 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nAsia](/news/articles/c28ejl4nvgyo)\\\\n\\\\n[Auction for John Lennon glasses and Abbey Road photos\\\\\\\\\\\\n-----------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n1 day ago\\\\\\\\\\\\n\\\\\\\\\\\\nSurrey](/news/articles/cp085ym9l3ro)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![A picture of Bella Hadid](https://ichef.bbci.co.uk/news/480/cpsprodpb/da1b/live/4c41b850-4678-11ef-895c-59d6554481d1.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nBella Hadid\\'s Adidas advert dropped after Israeli criticism\\\\\\\\\\\\n-----------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nThe model was chosen to promote shoes referencing the 1972 Munich Olympics, at which 11 Israeli athletes were killed.\\\\\\\\\\\\n\\\\\\\\\\\\n2 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nCulture](/news/articles/ceqdwpv8vw1o)\\\\n\\\\n* * *\\\\n\\\\nTravel\\\\n------\\\\n\\\\n[The Indian dish Kamala Harris loves\\\\\\\\\\\\n-----------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n1 hr ago\\\\\\\\\\\\n\\\\\\\\\\\\nTravel](/travel/article/20201026-dosa-indias-wholesome-fast-food-obsession)\\\\n\\\\n[Italy\\'s most iconic trail reopens after 12 years\\\\\\\\\\\\n------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n3 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nTravel](/travel/article/20240722-cinque-terre-italys-path-of-love-reopens-after-12-years)\\\\n\\\\n[The US\\'s little-known \\'Ellis Island of the West\\'\\\\\\\\\\\\n------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n5 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nTravel](/travel/article/20240719-angel-island-the-little-known-ellis-island-of-the-west-us)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![Dondurma (Credit: Alamy)](credit: Alamy)\\\\\\\\\\\\n\\\\\\\\\\\\nThe Turkish ice cream eaten with a knife and fork\\\\\\\\\\\\n-------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nDondurma isn\\'t like any other ice cream you\\'ll find, and the epicentre of its production is still reeling from the powerful earthquakes that decimated the nation.\\\\\\\\\\\\n\\\\\\\\\\\\n1 day ago\\\\\\\\\\\\n\\\\\\\\\\\\nTravel](/travel/article/20240719-dondurma-the-turkish-ice-cream-eaten-with-a-knife-and-fork)\\\\n\\\\n* * *\\\\n\\\\n[Travel\\\\\\\\\\\\n------](https://www.bbc.com/travel)\\\\n\\\\u00a0\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![Angel Island (Credit: Alamy)](credit: Alamy)\\\\\\\\\\\\n\\\\\\\\\\\\nThe US\\'s little-known \\'Ellis Island of the West\\'\\\\\\\\\\\\n------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nThis former quarantine and military station once processed as many as one million immigrants. Now, the picturesque island is one of the San Francisco Bay Area\\'s best urban getaways.\\\\\\\\\\\\n\\\\\\\\\\\\nSee more](/travel/article/20240719-angel-island-the-little-known-ellis-island-of-the-west-us)\\\\n\\\\n* * *\\\\n\\\\nSign up for our newsletters\\\\n---------------------------\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![The Essential List](https://ichef.bbci.co.uk/images/ic/1920x1080/p0h74xp9.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nThe Essential List\\\\\\\\\\\\n------------------\\\\\\\\\\\\n\\\\\\\\\\\\nThe week\\'s best stories, handpicked by BBC editors, in your inbox every Tuesday and Friday.](https://cloud.email.bbc.com/SignUp10_08?&at_bbc_team=studios&at_medium=display&at_objective=acquisition&at_ptr_type=&at_ptr_name=bbc.comhp&at_format=Module&at_link_origin=homepage&at_campaign=essentiallist&at_campaign_type=owned)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![US Election Unspun](https://ichef.bbci.co.uk/images/ic/1920x1080/p0h74xqg.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nUS Election Unspun\\\\\\\\\\\\n------------------\\\\\\\\\\\\n\\\\\\\\\\\\nCut through the spin with North America correspondent Anthony Zurcher - in your inbox every Wednesday.](https://cloud.email.bbc.com/US_Election_Unspun_newsletter_signup?&at_bbc_team=studios&at_medium=display&at_objective=acquisition&at_ptr_type=&at_ptr_name=bbc.comhp&at_format=Module&at_link_origin=homepage&at_campaign=uselectionunspun&at_campaign_type=owned)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![Medal Moments](https://ichef.bbci.co.uk/images/ic/raw/p0j53vjd.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nMedal Moments\\\\\\\\\\\\n-------------\\\\\\\\\\\\n\\\\\\\\\\\\nYour global guide to the Paris Olympics, from key highlights to heroic stories, daily to your inbox throughout the Games.](https://cloud.email.bbc.com/medalmoments_newsletter_signup?&at_bbc_team=studios&at_medium=emails&at_objective=acquisition&at_ptr_type=&at_ptr_name=bbc.com&at_format=Module&at_link_origin=homepage&at_campaign=olympics2024&at_campaign_id=&at_adset_name=&at_adset_id=&at_creation=&at_creative_id=&at_campaign_type=owned)\\\\n\\\\n* * *\\\\n\\\\nEarth\\\\n-----\\\\n\\\\n[The \\'Paddington bears\\' facing climate threat\\\\\\\\\\\\n--------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n2 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nNatural wonders](/reel/video/p0jbv222/climate-chaos-makes-paddington-bear-hangry-)\\\\n\\\\n[The simple Japanese method for a tidier fridge\\\\\\\\\\\\n----------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n3 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nFuture](/future/article/20240715-the-simple-japanese-method-for-a-tidier-and-less-wasteful-fridge)\\\\n\\\\n[Conspiracy theories swirl about geo-engineering, but could it help save the planet?\\\\\\\\\\\\n-----------------------------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n2 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nBBC InDepth](/news/articles/c98qp79gj4no)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![Humpback whale](https://ichef.bbci.co.uk/images/ic/480x270/p0jbqk2h.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\nFace to face with humpback whales\\\\\\\\\\\\n---------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nReece Parkinson discovers how locals are protecting their stunning marine environment.\\\\\\\\\\\\n\\\\\\\\\\\\n3 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nTravel](/reel/video/p0jbpt60/face-to-face-with-humpback-whales)\\\\n\\\\n* * *\\\\n\\\\nScience and health\\\\n------------------\\\\n\\\\n[India alert after boy dies from Nipah virus in Kerala\\\\\\\\\\\\n-----------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n6 hrs ago\\\\\\\\\\\\n\\\\\\\\\\\\nAsia](/news/articles/cj50d7e9vp6o)\\\\n\\\\n[Are fermented foods actually good for our health?\\\\\\\\\\\\n-------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n2 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nFuture](/future/article/20240719-are-fermented-foods-actually-good-for-you)\\\\n\\\\n[Summer surge: why Covid-19 isn\\'t yet seasonal\\\\\\\\\\\\n---------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n1 day ago\\\\\\\\\\\\n\\\\\\\\\\\\nFuture](/future/article/20240719-why-covid-19-is-spreading-this-summer)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![A young woman\\'s face with a red background](https://ichef.bbci.co.uk/news/1536/cpsprodpb/3884/live/affa3d80-45e3-11ef-837a-9936fb8608b6.jpg.webp)\\\\\\\\\\\\n\\\\\\\\\\\\n\\'A way to fight back\\': FGM survivors reclaim vulvas with surgery\\\\\\\\\\\\n----------------------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nIn Somalia, it\\'s believed cutting off a girl\\'s outer genitalia will guarantee their virginity.\\\\\\\\\\\\n\\\\\\\\\\\\n2 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nAfrica](/news/articles/cyx0perl8yno)\\\\n\\\\n* * *\\\\n\\\\nWorld\\\\u2019s Table\\\\n-------------\\\\n\\\\n[The Turkish ice cream eaten with a knife and fork\\\\\\\\\\\\n-------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n1 day ago\\\\\\\\\\\\n\\\\\\\\\\\\nTravel](/travel/article/20240719-dondurma-the-turkish-ice-cream-eaten-with-a-knife-and-fork)\\\\n\\\\n[India\\'s cooling drinks to beat the heat\\\\\\\\\\\\n---------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n7 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nTravel](/travel/article/20240715-gond-katira-a-natural-way-to-cool-down-in-indias-scorching-summers)\\\\n\\\\n[The world\\'s biggest restaurant is coming to Paris\\\\\\\\\\\\n-------------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n9 Jul 2024\\\\\\\\\\\\n\\\\\\\\\\\\nTravel](/travel/article/20240708-paris-olympics-2024-worlds-biggest-restaurant)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![Plate of \\\\u00e7i\\\\u011f k\\\\u00f6fte made with raw meat (Credit: Paul Benjamin Osterlund)](credit: Paul Benjamin Osterlund)\\\\\\\\\\\\n\\\\\\\\\\\\nThe wild ceremonies of the Turkish \\'meatball\\'\\\\\\\\\\\\n---------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nOne of the country\\'s most popular fast-food items, \\\\u00e7i\\\\u011f k\\\\u00f6fte is traditionally associated with wild and rowdy gatherings in south-eastern Turkey.\\\\\\\\\\\\n\\\\\\\\\\\\n1 Jun 2024\\\\\\\\\\\\n\\\\\\\\\\\\nTravel](/travel/article/20240531-the-wild-ceremonies-surrounding-a-turkish-meatball)\\\\n\\\\n* * *\\\\n\\\\nThe Specialist\\\\n--------------\\\\n\\\\n[Guide to Helsinki\\'s happiest places\\\\\\\\\\\\n-----------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n2 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nTravel](/travel/article/20240718-a-happiness-hackers-guide-to-the-happiest-outdoor-places-in-helsinki)\\\\n\\\\n[A pastry chef\\'s favourite bakeries in Paris\\\\\\\\\\\\n-------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n5 days ago\\\\\\\\\\\\n\\\\\\\\\\\\nTravel](/travel/article/20240717-david-lebovitzs-ultimate-guide-to-the-best-bakeries-in-paris-right-now)\\\\n\\\\n[Chef Andrew Zimmern\\'s favourite US restaurants\\\\\\\\\\\\n----------------------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\n14 Jul 2024\\\\\\\\\\\\n\\\\\\\\\\\\nTravel](/travel/article/20240714-chef-andrew-zimmerns-favourite-us-restaurants)\\\\n\\\\n[![](https://www.bbc.com/bbcx/grey-placeholder.png)![Iceland in winter (Credit: Getty Images)](credit: Getty Images)\\\\\\\\\\\\n\\\\\\\\\\\\nA First Lady\\'s guide to Iceland\\\\\\\\\\\\n-------------------------------\\\\\\\\\\\\n\\\\\\\\\\\\nEliza Reid moved to Iceland 20 years ago for love and now she\\'s the First Lady. Here are her favourite ways to enjoy a \\\\\"chill\\\\\" Icelandic weekend.\\\\\\\\\\\\n\\\\\\\\\\\\n11 Jul 2024\\\\\\\\\\\\n\\\\\\\\\\\\nTravel](/travel/article/20240710-icelands-first-lady-takes-you-on-a-tour-of-her-super-chill-nation)\\\\n\\\\n* * *\\\\n\\\\n[British Broadcasting Corporation](/)\\\\n\\\\n* [Home](https://www.bbc.com/)\\\\n \\\\n* [News](/news)\\\\n \\\\n* [Sport](/sport)\\\\n \\\\n* [Business](/business)\\\\n \\\\n* [Innovation](/innovation)\\\\n \\\\n* [Culture](/culture)\\\\n \\\\n* [Travel](/travel)\\\\n \\\\n* [Earth](/future-planet)\\\\n \\\\n* [Video](/video)\\\\n \\\\n* [Live](/live)\\\\n \\\\n* [Audio](https://www.bbc.co.uk/sounds)\\\\n \\\\n* [Weather](https://www.bbc.com/weather)\\\\n \\\\n* [BBC Shop](https://shop.bbc.com/)\\\\n \\\\n\\\\nBBC in other languages\\\\n\\\\nFollow BBC on:\\\\n--------------\\\\n\\\\n* [Terms of Use](https://www.bbc.co.uk/usingthebbc/terms)\\\\n \\\\n* [About the BBC](https://www.bbc.co.uk/aboutthebbc)\\\\n \\\\n* [Privacy Policy](https://www.bbc.com/usingthebbc/privacy/)\\\\n \\\\n* [Cookies](https://www.bbc.com/usingthebbc/cookies/)\\\\n \\\\n* [Accessibility Help](https://www.bbc.co.uk/accessibility/)\\\\n \\\\n* [Contact the BBC](https://www.bbc.co.uk/contact)\\\\n \\\\n* [Advertise with us](https://www.bbc.com/advertisingcontact)\\\\n \\\\n* [Do not share or sell my info](https://www.bbc.com/usingthebbc/cookies/how-can-i-change-my-bbc-cookie-settings/)\\\\n \\\\n* [Contact technical support](https://www.bbc.com/contact-bbc-com-help)\\\\n \\\\n\\\\nCopyright 2024 BBC. All rights reserved.\\\\u00a0\\\\u00a0The _BBC_ is _not responsible for the content of external sites._\\\\u00a0[**Read about our approach to external linking.**](https://www.bbc.co.uk/editorialguidelines/guidance/feeds-and-links)\", \"html\": \"
Skip to content

Only from the BBC

Must watch

In History

Sign up for our newsletters

\", \"metadata\": {\"title\": \"BBC Home - Breaking News, World News, US News, Sports, Business, Innovation, Climate, Culture, Travel, Video & AudioBritish Broadcasting CorporationBritish Broadcasting Corporation\", \"description\": \"Visit BBC for trusted reporting on the latest world and US news, sports, business, climate, innovation, culture and much more.\", \"robots\": \"NOODP, NOYDIR\", \"ogTitle\": \"BBC Home - Breaking News, World News, US News, Sports, Business, Innovation, Climate, Culture, Travel, Video & Audio\", \"ogDescription\": \"Visit BBC for trusted reporting on the latest world and US news, sports, business, climate, innovation, culture and much more.\", \"ogLocaleAlternate\": [], \"sourceURL\": \"https://www.bbc.com\", \"pageStatusCode\": 200}, \"linksOnPage\": [\"https://www.bbc.com/\", \"https://www.bbc.com/watch-live-news\", \"https://www.bbc.com/news\", \"https://www.bbc.com/sport\", \"https://www.bbc.com/business\", \"https://www.bbc.com/innovation\", \"https://www.bbc.com/culture\", \"https://www.bbc.com/travel\", \"https://www.bbc.com/future-planet\", \"https://www.bbc.com/video\", \"https://www.bbc.com/live\", \"https://www.bbc.co.uk/sounds\", \"https://www.bbc.com/weather\", \"https://www.bbc.com/newsletters\", \"https://www.bbc.com/news/articles/c250zqgrpqgo\", \"https://www.bbc.com/news/articles/c3gdzmdje5xo\", \"https://www.bbc.com/news/live/cv2gryx1yx1t\", \"https://www.bbc.com/news/articles/cpwd8yzw45qo\", \"https://www.bbc.com/news/articles/c80ekdwk9zro\", \"https://www.bbc.com/news/live/c724wqpy4qnt\", \"https://www.bbc.com/news/articles/c16j896003xo\", \"https://www.bbc.com/future/article/20240722-why-you-are-probably-sitting-down-for-too-long\", \"https://www.bbc.com/news/articles/c51yqdy3q61o\", \"https://www.bbc.com/news/articles/crgk8gnpg0zo\", \"https://www.bbc.com/news/articles/cgerz8we1vgo\", \"https://www.bbc.com/news/articles/cw4yxd3dw7qo\", \"https://www.bbc.com/sport/formula1/articles/cg3jzy3e8q1o\", \"https://www.bbc.com/news/articles/cj50d7e9vp6o\", \"https://www.bbc.com/news/articles/czrj18yp489o\", \"https://www.bbc.com/news/articles/cgl7e33n1d0o\", \"https://www.bbc.com/news/articles/cq5jwyel12qo\", \"https://www.bbc.com/news/videos/cx028eq4qg1o\", \"https://www.bbc.com/news/videos/cmj24x03r8po\", \"https://www.bbc.com/news/videos/c51yrr2z74no\", \"https://www.bbc.com/reel/video/p0jch19q/turkey-s-answer-to-burning-man-\", \"https://www.bbc.com/sport/basketball/videos/cg640yk3n3zo\", \"https://www.bbc.com/news/videos/cqe6q917y1jo\", \"https://www.bbc.com/news/videos/c0jq593xqk8o\", \"https://www.bbc.com/news/videos/cjk325014g7o\", \"https://www.bbc.com/culture/article/20240718-a-ww1-soldier-on-the-brutality-of-conflict\", \"https://www.bbc.com/sport/olympics\", \"https://www.bbc.com/sport/football/articles/cek91m98g48o\", \"https://www.bbc.com/news/articles/crgk8rgm87lo\", \"https://www.bbc.com/news/articles/cq5xdq71drro\", \"https://www.bbc.com/news/videos/c51y936y9g2o\", \"https://www.bbc.com/news/articles/cn08d7vyj6wo\", \"https://www.bbc.com/news/articles/cn08d7j1qj5o\", \"https://www.bbc.com/news/articles/cd1e48677w0o\", \"https://www.bbc.com/news/articles/cxw29lg4y8go\", \"https://www.bbc.com/news/articles/c4ngrdpwlx3o\", \"https://www.bbc.com/news/articles/cxr2g7rk3dxo\", \"https://www.bbc.com/sport/football/articles/c9e950ev0xpo\", \"https://www.bbc.com/sport/football/articles/cgrlgzx6gpro\", \"https://www.bbc.com/sport/articles/c2v0j0j6nqlo\", \"https://www.bbc.com/sport/golf/articles/cv2g1glwypqo\", \"https://www.bbc.com/reel/video/p0jbv222/climate-chaos-makes-paddington-bear-hangry-\", \"https://www.bbc.com/news/articles/cj50d6q3jlro\", \"https://www.bbc.com/news/articles/c4ng5q4jd62o\", \"https://www.bbc.com/news/articles/c4ng785gjv0o\", \"https://www.bbc.com/news/articles/cpe3zgznwjno\", \"https://www.bbc.com/news/articles/cp9vdxwjddeo\", \"https://www.bbc.com/future/article/20240719-why-covid-19-is-spreading-this-summer\", \"https://www.bbc.com/news/articles/cq5xy12pynyo\", \"https://www.bbc.com/future/article/20240719-are-fermented-foods-actually-good-for-you\", \"https://www.bbc.com/culture/article/20240719-how-to-choose-the-best-and-most-eco-friendly-swimwear\", \"https://www.bbc.com/news/articles/c28ejl4nvgyo\", \"https://www.bbc.com/news/articles/cp085ym9l3ro\", \"https://www.bbc.com/news/articles/ceqdwpv8vw1o\", \"https://www.bbc.com/travel/article/20201026-dosa-indias-wholesome-fast-food-obsession\", \"https://www.bbc.com/travel/article/20240722-cinque-terre-italys-path-of-love-reopens-after-12-years\", \"https://www.bbc.com/travel/article/20240719-angel-island-the-little-known-ellis-island-of-the-west-us\", \"https://www.bbc.com/travel/article/20240719-dondurma-the-turkish-ice-cream-eaten-with-a-knife-and-fork\", \"https://cloud.email.bbc.com/SignUp10_08?&at_bbc_team=studios&at_medium=display&at_objective=acquisition&at_ptr_type=&at_ptr_name=bbc.comhp&at_format=Module&at_link_origin=homepage&at_campaign=essentiallist&at_campaign_type=owned\", \"https://cloud.email.bbc.com/US_Election_Unspun_newsletter_signup?&at_bbc_team=studios&at_medium=display&at_objective=acquisition&at_ptr_type=&at_ptr_name=bbc.comhp&at_format=Module&at_link_origin=homepage&at_campaign=uselectionunspun&at_campaign_type=owned\", \"https://cloud.email.bbc.com/medalmoments_newsletter_signup?&at_bbc_team=studios&at_medium=emails&at_objective=acquisition&at_ptr_type=&at_ptr_name=bbc.com&at_format=Module&at_link_origin=homepage&at_campaign=olympics2024&at_campaign_id=&at_adset_name=&at_adset_id=&at_creation=&at_creative_id=&at_campaign_type=owned\", \"https://www.bbc.com/future/article/20240715-the-simple-japanese-method-for-a-tidier-and-less-wasteful-fridge\", \"https://www.bbc.com/news/articles/c98qp79gj4no\", \"https://www.bbc.com/reel/video/p0jbpt60/face-to-face-with-humpback-whales\", \"https://www.bbc.com/news/articles/cyx0perl8yno\", \"https://www.bbc.com/travel/article/20240715-gond-katira-a-natural-way-to-cool-down-in-indias-scorching-summers\", \"https://www.bbc.com/travel/article/20240708-paris-olympics-2024-worlds-biggest-restaurant\", \"https://www.bbc.com/travel/article/20240531-the-wild-ceremonies-surrounding-a-turkish-meatball\", \"https://www.bbc.com/travel/article/20240718-a-happiness-hackers-guide-to-the-happiest-outdoor-places-in-helsinki\", \"https://www.bbc.com/travel/article/20240717-david-lebovitzs-ultimate-guide-to-the-best-bakeries-in-paris-right-now\", \"https://www.bbc.com/travel/article/20240714-chef-andrew-zimmerns-favourite-us-restaurants\", \"https://www.bbc.com/travel/article/20240710-icelands-first-lady-takes-you-on-a-tour-of-her-super-chill-nation\", \"https://shop.bbc.com/\", \"https://www.bbc.co.uk/usingthebbc/terms\", \"https://www.bbc.co.uk/aboutthebbc\", \"https://www.bbc.com/usingthebbc/privacy/\", \"https://www.bbc.com/usingthebbc/cookies/\", \"https://www.bbc.co.uk/accessibility/\", \"https://www.bbc.co.uk/contact\", \"https://www.bbc.com/advertisingcontact\", \"https://www.bbc.com/usingthebbc/cookies/how-can-i-change-my-bbc-cookie-settings/\", \"https://www.bbc.com/contact-bbc-com-help\", \"https://www.bbc.co.uk/editorialguidelines/guidance/feeds-and-links\"]}, \"returnCode\": 200}', role=, name=None, meta={})]}}" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "query = \"What is on the front-page of BBC today?\"\n", - "\n", - "pipe.run({\"prompt_builder\": {\"query\": query}, \"router\": {\"query\": query}})" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/haystack_experimental/components/__init__.py b/haystack_experimental/components/__init__.py index 7eaba976..3d12e900 100644 --- a/haystack_experimental/components/__init__.py +++ b/haystack_experimental/components/__init__.py @@ -10,7 +10,7 @@ from .retrievers.auto_merging_retriever import AutoMergingRetriever from .retrievers.chat_message_retriever import ChatMessageRetriever from .splitters import HierarchicalDocumentSplitter -from .tools import OpenAIFunctionCaller, ToolInvoker +from .tools import ToolInvoker from .writers import ChatMessageWriter _all_ = [ @@ -23,6 +23,5 @@ "AnthropicChatGenerator", "LLMMetadataExtractor", "HierarchicalDocumentSplitter", - "OpenAIFunctionCaller", "ToolInvoker", ] diff --git a/haystack_experimental/components/tools/__init__.py b/haystack_experimental/components/tools/__init__.py index 31a4e4f3..24338e8c 100644 --- a/haystack_experimental/components/tools/__init__.py +++ b/haystack_experimental/components/tools/__init__.py @@ -2,7 +2,6 @@ # # SPDX-License-Identifier: Apache-2.0 -from .openai.function_caller import OpenAIFunctionCaller from .tool_invoker import ToolInvoker -_all_ = ["OpenAIFunctionCaller", "ToolInvoker"] +_all_ = ["ToolInvoker"] diff --git a/haystack_experimental/components/tools/openai/__init__.py b/haystack_experimental/components/tools/openai/__init__.py deleted file mode 100644 index 1c3f4517..00000000 --- a/haystack_experimental/components/tools/openai/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 - -from .function_caller import OpenAIFunctionCaller - -_all_ = [ "OpenAIFunctionCaller"] diff --git a/haystack_experimental/components/tools/openai/function_caller.py b/haystack_experimental/components/tools/openai/function_caller.py deleted file mode 100644 index c4e1a573..00000000 --- a/haystack_experimental/components/tools/openai/function_caller.py +++ /dev/null @@ -1,102 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 - -import json -from typing import Any, Callable, Dict, List - -from haystack import component, default_from_dict, default_to_dict -from haystack.dataclasses import ChatMessage -from haystack.utils import deserialize_callable, serialize_callable - -_FUNCTION_NAME_FAILURE = ( - "I'm sorry, I tried to run a function that did not exist. Would you like me to correct it and try again?" -) -_FUNCTION_RUN_FAILURE = "Seems there was an error while running the function: {error}" - - -@component -class OpenAIFunctionCaller: - """ - OpenAIFunctionCaller processes a list of chat messages and call Python functions when needed. - - The OpenAIFunctionCaller expects a list of ChatMessages and if there is a tool call with a function name and - arguments, it runs the function and returns the result as a ChatMessage from role = 'function' - """ - - def __init__(self, available_functions: Dict[str, Callable]): - """ - Initialize the OpenAIFunctionCaller component. - - :param available_functions: - A dictionary of available functions. This dictionary expects key value pairs of function name, - and the function itself. For example, `{"weather_function": weather_function}` - """ - self.available_functions = available_functions - - def to_dict(self) -> Dict[str, Any]: - """ - Serializes the component to a dictionary. - - :returns: - Dictionary with serialized data. - """ - available_function_paths = {} - for name, function in self.available_functions.items(): - available_function_paths[name] = serialize_callable(function) - serialization_dict = default_to_dict(self, available_functions=available_function_paths) - return serialization_dict - - @classmethod - def from_dict(cls, data: Dict[str, Any]) -> "OpenAIFunctionCaller": - """ - Deserializes the component from a dictionary. - - :param data: - The dictionary to deserialize from. - :returns: - The deserialized component. - """ - available_function_paths = data.get("init_parameters", {}).get("available_functions") - available_functions = {} - for name, path in available_function_paths.items(): - available_functions[name] = deserialize_callable(path) - data["init_parameters"]["available_functions"] = available_functions - return default_from_dict(cls, data) - - @component.output_types(function_replies=List[ChatMessage], assistant_replies=List[ChatMessage]) - def run(self, messages: List[ChatMessage]): - """ - Evaluates `messages` and invokes available functions if the messages contain tool_calls. - - :param messages: A list of messages generated from the `OpenAIChatGenerator` - :returns: This component returns a list of messages in one of two outputs - - `function_replies`: List of ChatMessages containing the result of a function invocation. - This message comes from role = 'function'. If the function name was hallucinated or wrong, - an assistant message explaining as such is returned - - `assistant_replies`: List of ChatMessages containing a regular assistant reply. In this case, - there were no tool_calls in the received messages - """ - if messages[0].meta["finish_reason"] == "tool_calls": - function_calls = json.loads(messages[0].content) - for function_call in function_calls: - function_name = function_call["function"]["name"] - function_args = json.loads(function_call["function"]["arguments"]) - if function_name in self.available_functions: - function_to_call = self.available_functions[function_name] - try: - function_response = function_to_call(**function_args) - messages.append( - # We disable ensure_ascii so special chars like emojis are not converted - ChatMessage.from_function( - content=json.dumps(function_response, ensure_ascii=False), - name=function_name, - ) - ) - # pylint: disable=broad-exception-caught - except Exception as e: - messages.append(ChatMessage.from_assistant(_FUNCTION_RUN_FAILURE.format(error=e))) - else: - messages.append(ChatMessage.from_assistant(_FUNCTION_NAME_FAILURE)) - return {"function_replies": messages} - return {"assistant_replies": messages} diff --git a/haystack_experimental/components/tools/openapi/__init__.py b/haystack_experimental/components/tools/openapi/__init__.py deleted file mode 100644 index 6867e62b..00000000 --- a/haystack_experimental/components/tools/openapi/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 - -from haystack_experimental.components.tools.openapi.openapi_tool import OpenAPITool -from haystack_experimental.components.tools.openapi.types import LLMProvider - -__all__ = ["LLMProvider", "OpenAPITool"] diff --git a/haystack_experimental/components/tools/openapi/_openapi.py b/haystack_experimental/components/tools/openapi/_openapi.py deleted file mode 100644 index 5528af5e..00000000 --- a/haystack_experimental/components/tools/openapi/_openapi.py +++ /dev/null @@ -1,330 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 - -import logging -from collections import defaultdict -from typing import Any, Callable, Dict, List, Optional - -import requests - -from haystack_experimental.components.tools.openapi._payload_extraction import ( - create_function_payload_extractor, -) -from haystack_experimental.components.tools.openapi._schema_conversion import ( - anthropic_converter, - cohere_converter, - openai_converter, -) -from haystack_experimental.components.tools.openapi.types import LLMProvider, OpenAPISpecification, Operation -from haystack_experimental.components.tools.utils import normalize_tool_definition - -MIN_REQUIRED_OPENAPI_SPEC_VERSION = 3 -logger = logging.getLogger(__name__) - - -def send_request(request: Dict[str, Any]) -> Dict[str, Any]: - """ - Send an HTTP request and return the response. - - :param request: The request to send. - :returns: The response from the server. - """ - url = request["url"] - headers = {**request.get("headers", {})} - try: - response = requests.request( - request["method"], - url, - headers=headers, - params=request.get("params", {}), - json=request.get("json"), - auth=request.get("auth"), - timeout=30, - ) - response.raise_for_status() - return response.json() - except requests.exceptions.HTTPError as e: - logger.warning("HTTP error occurred: %s while sending request to %s", e, url) - raise HttpClientError(f"HTTP error occurred: {e}") from e - except requests.exceptions.RequestException as e: - logger.warning("Request error occurred: %s while sending request to %s", e, url) - raise HttpClientError(f"HTTP error occurred: {e}") from e - except Exception as e: - logger.warning("An error occurred: %s while sending request to %s", e, url) - raise HttpClientError(f"An error occurred: {e}") from e - - -# Authentication strategies -def create_api_key_auth_function(api_key: str) -> Callable[[Dict[str, Any], Dict[str, Any]], None]: - """ - Create a function that applies the API key authentication strategy to a given request. - - :param api_key: the API key to use for authentication. - :returns: a function that applies the API key authentication to a request - at the schema specified location. - """ - - def apply_auth(security_scheme: Dict[str, Any], request: Dict[str, Any]) -> None: - """ - Apply the API key authentication strategy to the given request. - - :param security_scheme: the security scheme from the OpenAPI spec. - :param request: the request to apply the authentication to. - """ - if security_scheme["in"] == "header": - request.setdefault("headers", {})[security_scheme["name"]] = api_key - elif security_scheme["in"] == "query": - request.setdefault("params", {})[security_scheme["name"]] = api_key - elif security_scheme["in"] == "cookie": - request.setdefault("cookies", {})[security_scheme["name"]] = api_key - else: - raise ValueError( - f"Unsupported apiKey authentication location: {security_scheme['in']}, " - f"must be one of 'header', 'query', or 'cookie'" - ) - - return apply_auth - - -def create_http_auth_function(token: str) -> Callable[[Dict[str, Any], Dict[str, Any]], None]: - """ - Create a function that applies the http authentication strategy to a given request. - - :param token: the authentication token to use. - :returns: a function that applies the API key authentication to a request - at the schema specified location. - """ - - def apply_auth(security_scheme: Dict[str, Any], request: Dict[str, Any]) -> None: - """ - Apply the HTTP authentication strategy to the given request. - - :param security_scheme: the security scheme from the OpenAPI spec. - :param request: the request to apply the authentication to. - """ - if security_scheme["type"] == "http": - # support bearer http auth, no basic support yet - if security_scheme["scheme"].lower() == "bearer": - if not token: - raise ValueError("Token must be provided for Bearer Auth.") - request.setdefault("headers", {})["Authorization"] = f"Bearer {token}" - else: - raise ValueError(f"Unsupported HTTP authentication scheme: {security_scheme['scheme']}") - else: - raise ValueError("HTTPAuthentication strategy received a non-HTTP security scheme.") - - return apply_auth - - -class HttpClientError(Exception): - """Exception raised for errors in the HTTP client.""" - - -class ClientConfiguration: - """Configuration for the OpenAPI client.""" - - def __init__( - self, - openapi_spec: OpenAPISpecification, - credentials: Optional[str] = None, - request_sender: Optional[Callable[[Dict[str, Any]], Dict[str, Any]]] = None, - llm_provider: LLMProvider = LLMProvider.OPENAI, - operations_filter: Optional[Callable[[Dict[str, Any]], bool]] = None, - ): # noqa: PLR0913 # pylint: disable=too-many-positional-arguments - """ - Initialize a ClientConfiguration instance. - - :param openapi_spec: The OpenAPI specification to use for the client. - :param credentials: The credentials to use for authentication. - :param request_sender: The function to use for sending requests. - :param llm_provider: The LLM provider to use for generating tools definitions. - :param operations_filter: A function to filter the functions to register with LLMs. - :raises ValueError: If the OpenAPI specification format is invalid. - """ - self.openapi_spec = openapi_spec - self.credentials = credentials - self.request_sender = request_sender or send_request - self.llm_provider: LLMProvider = llm_provider - self.operation_filter = operations_filter - - def get_auth_function(self) -> Callable[[Dict[str, Any], Dict[str, Any]], Any]: - """ - Get the authentication function that sets a schema specified authentication to the request. - - The function takes a security scheme and a request as arguments: - `security_scheme: Dict[str, Any] - The security scheme from the OpenAPI spec.` - `request: Dict[str, Any] - The request to apply the authentication to.` - :returns: The authentication function. - :raises ValueError: If the credentials type is not supported. - """ - security_schemes = self.openapi_spec.get_security_schemes() - if not self.credentials: - return lambda security_scheme, request: None # No-op function - if isinstance(self.credentials, str): - return self._create_authentication_from_string(self.credentials, security_schemes) - raise ValueError(f"Unsupported credentials type: {type(self.credentials)}") - - def get_tools_definitions(self) -> List[Dict[str, Any]]: - """ - Get the tools definitions used as tools LLM parameter. - - :returns: The tools definitions ready to be passed to the LLM as tools parameter. - """ - provider_to_converter = defaultdict( - lambda: openai_converter, - { - LLMProvider.ANTHROPIC: anthropic_converter, - LLMProvider.COHERE: cohere_converter, - }, - ) - converter = provider_to_converter[self.llm_provider] - tools_definitions = converter(self.openapi_spec, self.operation_filter) - return [normalize_tool_definition(t) for t in tools_definitions] - - def get_payload_extractor(self) -> Callable[[Dict[str, Any]], Dict[str, Any]]: - """ - Get the payload extractor for the LLM provider. - - This function knows how to extract the exact function payload from the LLM generated function calling payload. - :returns: The payload extractor function. - """ - provider_to_arguments_field_name = defaultdict( - lambda: "arguments", - { - LLMProvider.ANTHROPIC: "input", - LLMProvider.COHERE: "parameters", - }, - ) - arguments_field_name = provider_to_arguments_field_name[self.llm_provider] - return create_function_payload_extractor(arguments_field_name) - - def _create_authentication_from_string( - self, credentials: str, security_schemes: Dict[str, Any] - ) -> Callable[[Dict[str, Any], Dict[str, Any]], Any]: - for scheme in security_schemes.values(): - if scheme["type"] == "apiKey": - return create_api_key_auth_function(api_key=credentials) - if scheme["type"] == "http": - return create_http_auth_function(token=credentials) - raise ValueError(f"Unsupported authentication type '{scheme['type']}' provided.") - raise ValueError(f"Unable to create authentication from provided credentials: {credentials}") - - -def build_request(operation: Operation, **kwargs) -> Dict[str, Any]: - """ - Build an HTTP request for the operation. - - :param operation: The operation to build the request for. - :param kwargs: The arguments to use for building the request. - :returns: The HTTP request as a dictionary. - :raises ValueError: If a required parameter is missing. - :raises NotImplementedError: If the request body content type is not supported. We only support JSON payloads. - """ - path = operation.path - for parameter in operation.get_parameters("path"): - param_value = kwargs.get(parameter["name"], None) - if param_value: - path = path.replace(f"{{{parameter['name']}}}", str(param_value)) - elif parameter.get("required", False): - raise ValueError(f"Missing required path parameter: {parameter['name']}") - url = operation.get_server() + path - # method - method = operation.method.lower() - # headers - headers = {} - for parameter in operation.get_parameters("header"): - param_value = kwargs.get(parameter["name"], None) - if param_value: - headers[parameter["name"]] = str(param_value) - elif parameter.get("required", False): - raise ValueError(f"Missing required header parameter: {parameter['name']}") - # query params - query_params = {} - for parameter in operation.get_parameters("query"): - param_value = kwargs.get(parameter["name"], None) - if param_value: - query_params[parameter["name"]] = param_value - elif parameter.get("required", False): - raise ValueError(f"Missing required query parameter: {parameter['name']}") - - json_payload = None - request_body = operation.request_body - if request_body: - content = request_body.get("content", {}) - if "application/json" in content: - json_payload = {**kwargs} - else: - raise NotImplementedError("Request body content type not supported") - return { - "url": url, - "method": method, - "headers": headers, - "params": query_params, - "json": json_payload, - } - - -def apply_authentication( - auth_strategy: Callable[[Dict[str, Any], Dict[str, Any]], Any], - operation: Operation, - request: Dict[str, Any], -): - """ - Apply the authentication strategy to the given request. - - :param auth_strategy: The authentication strategy to apply. - This is a function that takes a security scheme and a request as arguments (at runtime) - and applies the authentication - :param operation: The operation to apply the authentication to. - :param request: The request to apply the authentication to. - """ - security_requirements = operation.security_requirements - security_schemes = operation.spec_dict.get("components", {}).get("securitySchemes", {}) - if security_requirements: - for requirement in security_requirements: - for scheme_name in requirement: - if scheme_name in security_schemes: - security_scheme = security_schemes[scheme_name] - auth_strategy(security_scheme, request) - break - - -class OpenAPIServiceClient: - """ - A client for invoking operations on REST services defined by OpenAPI specifications. - """ - - def __init__(self, client_config: ClientConfiguration): - self.client_config = client_config - - def invoke(self, function_payload: Any) -> Any: - """ - Invokes a function specified in the function payload. - - :param function_payload: The function payload containing the details of the function to be invoked. - :returns: The response from the service after invoking the function. - :raises OpenAPIClientError: If the function invocation payload cannot be extracted from the function payload. - :raises HttpClientError: If an error occurs while sending the request and receiving the response. - """ - fn_invocation_payload = {} - try: - fn_extractor = self.client_config.get_payload_extractor() - fn_invocation_payload = fn_extractor(function_payload) - except Exception as e: - raise OpenAPIClientError(f"Error extracting function invocation payload: {str(e)}") from e - - if "name" not in fn_invocation_payload or "arguments" not in fn_invocation_payload: - raise OpenAPIClientError( - f"Function invocation payload does not contain 'name' or 'arguments' keys: {fn_invocation_payload}, " - f"the payload extraction function may be incorrect." - ) - # fn_invocation_payload, if not empty, guaranteed to have "name" and "arguments" keys from here on - operation = self.client_config.openapi_spec.find_operation_by_id(fn_invocation_payload["name"]) - request = build_request(operation, **fn_invocation_payload["arguments"]) - apply_authentication(self.client_config.get_auth_function(), operation, request) - return self.client_config.request_sender(request) - - -class OpenAPIClientError(Exception): - """Exception raised for errors in the OpenAPI client.""" diff --git a/haystack_experimental/components/tools/openapi/_payload_extraction.py b/haystack_experimental/components/tools/openapi/_payload_extraction.py deleted file mode 100644 index 61bb1abf..00000000 --- a/haystack_experimental/components/tools/openapi/_payload_extraction.py +++ /dev/null @@ -1,86 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 - -import dataclasses -import json -from typing import Any, Callable, Dict, List, Optional, Union, cast - - -def create_function_payload_extractor( - arguments_field_name: str, -) -> Callable[[Any], Dict[str, Any]]: - """ - Extracts invocation payload from a given LLM completion containing function invocation. - - :param arguments_field_name: The name of the field containing the function arguments. - :return: A function that extracts the function invocation details from the LLM payload. - """ - - def _extract_function_invocation(payload: Any) -> Dict[str, Any]: - """ - Extract the function invocation details from the payload. - - :param payload: The LLM fc payload to extract the function invocation details from. - """ - fields_and_values = _search(payload, arguments_field_name) - if fields_and_values: - arguments = fields_and_values.get(arguments_field_name) - if not isinstance(arguments, (str, dict)): - raise ValueError( - f"Invalid {arguments_field_name} type {type(arguments)} for function call, expected str/dict" - ) - return { - "name": fields_and_values.get("name"), - "arguments": ( - json.loads(arguments) if isinstance(arguments, str) else arguments - ), - } - return {} - - return _extract_function_invocation - - -def _get_dict_converter( - obj: Any, method_names: Optional[List[str]] = None -) -> Union[Callable[[], Dict[str, Any]], None]: - method_names = method_names or [ - "model_dump", - "dict", - ] # search for pydantic v2 then v1 - for attr in method_names: - if hasattr(obj, attr) and callable(getattr(obj, attr)): - return getattr(obj, attr) - return None - - -def _is_primitive(obj) -> bool: - return isinstance(obj, (int, float, str, bool, type(None))) - - -def _required_fields(arguments_field_name: str) -> List[str]: - return ["name", arguments_field_name] - - -def _search(payload: Any, arguments_field_name: str) -> Dict[str, Any]: - if _is_primitive(payload): - return {} - if dict_converter := _get_dict_converter(payload): - payload = dict_converter() - elif dataclasses.is_dataclass(payload): - # Cast payload to Any to satisfy mypy 1.11.0 - payload = dataclasses.asdict(cast(Any, payload)) - if isinstance(payload, dict): - if all(field in payload for field in _required_fields(arguments_field_name)): - # this is the payload we are looking for - return payload - for value in payload.values(): - result = _search(value, arguments_field_name) - if result: - return result - elif isinstance(payload, list): - for item in payload: - result = _search(item, arguments_field_name) - if result: - return result - return {} diff --git a/haystack_experimental/components/tools/openapi/_schema_conversion.py b/haystack_experimental/components/tools/openapi/_schema_conversion.py deleted file mode 100644 index a5fb154e..00000000 --- a/haystack_experimental/components/tools/openapi/_schema_conversion.py +++ /dev/null @@ -1,306 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 - -import logging -from typing import Any, Callable, Dict, List, Optional - -from haystack_experimental.components.tools.openapi.types import ( - VALID_HTTP_METHODS, - OpenAPISpecification, - path_to_operation_id, -) - -MIN_REQUIRED_OPENAPI_SPEC_VERSION = 3 - -logger = logging.getLogger(__name__) - - -def openai_converter( - schema: OpenAPISpecification, - operation_filter: Optional[Callable[[Dict[str, Any]], bool]] = None, -) -> List[Dict[str, Any]]: - """ - Converts OpenAPI specification to a list of function suitable for OpenAI LLM function calling. - - See https://platform.openai.com/docs/guides/function-calling for more information about OpenAI's function schema. - :param schema: The OpenAPI specification to convert. - :param operation_filter: A function to filter operations to register with LLMs. - :returns: A list of dictionaries, each dictionary representing an OpenAI function definition. - """ - fn_definitions = _openapi_to_functions( - schema.spec_dict, "parameters", _parse_endpoint_spec_openai, operation_filter - ) - return [{"type": "function", "function": fn} for fn in fn_definitions] - - -def anthropic_converter( - schema: OpenAPISpecification, - operation_filter: Optional[Callable[[Dict[str, Any]], bool]] = None, -) -> List[Dict[str, Any]]: - """ - Converts an OpenAPI specification to a list of function definitions for Anthropic LLM function calling. - - See https://docs.anthropic.com/en/docs/tool-use for more information about Anthropic's function schema. - - :param schema: The OpenAPI specification to convert. - :param operation_filter: A function to filter operations to register with LLMs. - :returns: A list of dictionaries, each dictionary representing Anthropic function definition. - """ - - return _openapi_to_functions( - schema.spec_dict, "input_schema", _parse_endpoint_spec_openai, operation_filter - ) - - -def cohere_converter( - schema: OpenAPISpecification, - operation_filter: Optional[Callable[[Dict[str, Any]], bool]] = None, -) -> List[Dict[str, Any]]: - """ - Converts an OpenAPI specification to a list of function definitions for Cohere LLM function calling. - - See https://docs.cohere.com/docs/tool-use for more information about Cohere's function schema. - - :param schema: The OpenAPI specification to convert. - :param operation_filter: A function to filter operations to register with LLMs. - :returns: A list of dictionaries, each representing a Cohere style function definition. - """ - return _openapi_to_functions( - schema.spec_dict,"not important for cohere",_parse_endpoint_spec_cohere, operation_filter - ) - - -def _openapi_to_functions( - service_openapi_spec: Dict[str, Any], - parameters_name: str, - parse_endpoint_fn: Callable[[Dict[str, Any], str], Dict[str, Any]], - operation_filter: Optional[Callable[[Dict[str, Any]], bool]] = None, -) -> List[Dict[str, Any]]: - """ - Extracts operations from the OpenAPI specification, converts them into a function schema. - - :param service_openapi_spec: The OpenAPI specification to extract operations from. - :param parameters_name: The name of the parameters field in the function schema. - :param parse_endpoint_fn: The function to parse the endpoint specification. - :param operation_filter: A function to filter operations to register with LLMs. - :returns: A list of dictionaries, each dictionary representing a function schema. - """ - - # Doesn't enforce rigid spec validation because that would require a lot of dependencies - # We check the version and require minimal fields to be present, so we can extract operations - spec_version = service_openapi_spec.get("openapi") - if not spec_version: - raise ValueError( - f"Invalid OpenAPI spec provided. Could not extract version from {service_openapi_spec}" - ) - service_openapi_spec_version = int(spec_version.split(".")[0]) - # Compare the versions - if service_openapi_spec_version < MIN_REQUIRED_OPENAPI_SPEC_VERSION: - raise ValueError( - f"Invalid OpenAPI spec version {service_openapi_spec_version}. Must be " - f"at least {MIN_REQUIRED_OPENAPI_SPEC_VERSION}." - ) - operations: List[Dict[str, Any]] = [] - for path, path_value in service_openapi_spec["paths"].items(): - for path_key, operation_spec in path_value.items(): - if path_key.lower() in VALID_HTTP_METHODS: - if "operationId" not in operation_spec: - operation_spec["operationId"] = path_to_operation_id(path, path_key) - - # Apply the filter based on operationId before parsing the endpoint (operation) - if operation_filter and not operation_filter(operation_spec): - continue - - # parse (and register) this operation as it passed the filter - ops_dict = parse_endpoint_fn(operation_spec, parameters_name) - if ops_dict: - operations.append(ops_dict) - return operations - - -def _parse_endpoint_spec_openai( - resolved_spec: Dict[str, Any], parameters_name: str -) -> Dict[str, Any]: - """ - Parses an OpenAPI endpoint specification for OpenAI. - - :param resolved_spec: The resolved OpenAPI specification. - :param parameters_name: The name of the parameters field in the function schema. - :returns: A dictionary containing the parsed function schema. - """ - if not isinstance(resolved_spec, dict): - logger.warning( - "Invalid OpenAPI spec format provided. Could not extract function." - ) - return {} - function_name = resolved_spec.get("operationId") - description = resolved_spec.get("description") or resolved_spec.get("summary", "") - schema: Dict[str, Any] = {"type": "object", "properties": {}} - # requestBody section - req_body_schema = ( - resolved_spec.get("requestBody", {}) - .get("content", {}) - .get("application/json", {}) - .get("schema", {}) - ) - if "properties" in req_body_schema: - for prop_name, prop_schema in req_body_schema["properties"].items(): - schema["properties"][prop_name] = _parse_property_attributes(prop_schema) - if "required" in req_body_schema: - schema.setdefault("required", []).extend(req_body_schema["required"]) - - # parameters section - for param in resolved_spec.get("parameters", []): - if "schema" in param: - schema_dict = _parse_property_attributes(param["schema"]) - # these attributes are not in param[schema] level but on param level - useful_attributes = ["description", "pattern", "enum"] - schema_dict.update( - {key: param[key] for key in useful_attributes if param.get(key)} - ) - schema["properties"][param["name"]] = schema_dict - if param.get("required", False): - schema.setdefault("required", []).append(param["name"]) - - if function_name and description and schema["properties"]: - return { - "name": function_name, - "description": description, - parameters_name: schema, - } - logger.warning( - "Invalid OpenAPI spec format provided. Could not extract function from %s", - resolved_spec, - ) - return {} - - -def _parse_property_attributes( - property_schema: Dict[str, Any], include_attributes: Optional[List[str]] = None -) -> Dict[str, Any]: - """ - Recursively parses the attributes of a property schema. - - :param property_schema: The property schema to parse. - :param include_attributes: The attributes to include in the parsed schema. - :returns: A dictionary containing the parsed property schema. - """ - include_attributes = include_attributes or ["description", "pattern", "enum"] - schema_type = property_schema.get("type") - parsed_schema = {"type": schema_type} if schema_type else {} - for attr in include_attributes: - if attr in property_schema: - parsed_schema[attr] = property_schema[attr] - if schema_type == "object": - properties = property_schema.get("properties", {}) - parsed_properties = { - prop_name: _parse_property_attributes(prop, include_attributes) - for prop_name, prop in properties.items() - } - parsed_schema["properties"] = parsed_properties - if "required" in property_schema: - parsed_schema["required"] = property_schema["required"] - elif schema_type == "array": - items = property_schema.get("items", {}) - parsed_schema["items"] = _parse_property_attributes(items, include_attributes) - return parsed_schema - - -def _parse_endpoint_spec_cohere( - operation: Dict[str, Any], ignored_param: str -) -> Dict[str, Any]: - """ - Parses an endpoint specification for Cohere. - - :param operation: The operation specification to parse. - :param ignored_param: ignored, left for compatibility with the OpenAI converter. - :returns: A dictionary containing the parsed function schema. - """ - function_name = operation.get("operationId") - description = operation.get("description") or operation.get("summary", "") - parameter_definitions = _parse_parameters(operation) - if function_name: - return { - "name": function_name, - "description": description, - "parameter_definitions": parameter_definitions, - } - logger.warning("Operation missing operationId, cannot create function definition.") - return {} - - -def _parse_parameters(operation: Dict[str, Any]) -> Dict[str, Any]: - """ - Parses the parameters from an operation specification. - - :param operation: The operation specification to parse. - :returns: A dictionary containing the parsed parameters. - """ - parameters = {} - for param in operation.get("parameters", []): - if "schema" in param: - parameters[param["name"]] = _parse_schema( - param["schema"], - param.get("required", False), - param.get("description", ""), - ) - if "requestBody" in operation: - content = ( - operation["requestBody"].get("content", {}).get("application/json", {}) - ) - if "schema" in content: - schema_properties = content["schema"].get("properties", {}) - required_properties = content["schema"].get("required", []) - for name, schema in schema_properties.items(): - parameters[name] = _parse_schema( - schema, name in required_properties, schema.get("description", "") - ) - return parameters - - -def _parse_schema( - schema: Dict[str, Any], required: bool, description: str -) -> Dict[str, Any]: # noqa: FBT001 - """ - Parses a schema part of an operation specification. - - :param schema: The schema to parse. - :param required: Whether the schema is required. - :param description: The description of the schema. - :returns: A dictionary containing the parsed schema. - """ - schema_type = _get_type(schema) - if schema_type == "object": - # Recursive call for complex types - properties = schema.get("properties", {}) - nested_parameters = { - name: _parse_schema( - schema=prop_schema, - required=bool(name in schema.get("required", [])), - description=prop_schema.get("description", ""), - ) - for name, prop_schema in properties.items() - } - return { - "type": schema_type, - "description": description, - "properties": nested_parameters, - "required": required, - } - return {"type": schema_type, "description": description, "required": required} - - -def _get_type(schema: Dict[str, Any]) -> str: - type_mapping = { - "integer": "int", - "string": "str", - "boolean": "bool", - "number": "float", - "object": "object", - "array": "list", - } - schema_type = schema.get("type", "object") - if schema_type not in type_mapping: - raise ValueError(f"Unsupported schema type {schema_type}") - return type_mapping[schema_type] diff --git a/haystack_experimental/components/tools/openapi/openapi_tool.py b/haystack_experimental/components/tools/openapi/openapi_tool.py deleted file mode 100644 index 9c9b33a2..00000000 --- a/haystack_experimental/components/tools/openapi/openapi_tool.py +++ /dev/null @@ -1,225 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 - -import json -import os -from pathlib import Path -from typing import Any, Dict, List, Optional, Union - -from haystack import component, default_from_dict, default_to_dict, logging -from haystack.components.generators.chat import OpenAIChatGenerator -from haystack.dataclasses import ChatMessage, ChatRole -from haystack.lazy_imports import LazyImport -from haystack.utils import Secret, deserialize_secrets_inplace -from haystack.utils.url_validation import is_valid_http_url - -from haystack_experimental.components.tools.openapi._openapi import ( - ClientConfiguration, - OpenAPIServiceClient, -) -from haystack_experimental.components.tools.openapi.types import LLMProvider, OpenAPISpecification -from haystack_experimental.util import serialize_secrets_inplace - -with LazyImport("Run 'pip install anthropic-haystack'") as anthropic_import: - # pylint: disable=import-error - from haystack_integrations.components.generators.anthropic import AnthropicChatGenerator - -with LazyImport("Run 'pip install cohere-haystack'") as cohere_import: - # pylint: disable=import-error - from haystack_integrations.components.generators.cohere import CohereChatGenerator - -logger = logging.getLogger(__name__) - - -@component -class OpenAPITool: - """ - The OpenAPITool calls a RESTful endpoint of an OpenAPI service using payloads generated from human instructions. - - Here is an example of how to use the OpenAPITool component to scrape a URL using the FireCrawl API: - - ```python - from haystack.dataclasses import ChatMessage - from haystack_experimental.components.tools.openapi import OpenAPITool, LLMProvider - from haystack.utils import Secret - - tool = OpenAPITool(generator_api=LLMProvider.OPENAI, - generator_api_params={"model":"gpt-4o-mini"}, - spec="https://raw.githubusercontent.com/mendableai/firecrawl/main/apps/api/openapi.json", - credentials=Secret.from_env_var("FIRECRAWL_API_KEY")) - - results = tool.run(messages=[ChatMessage.from_user("Scrape URL: https://news.ycombinator.com/")]) - print(results) - ``` - - Similarly, you can use the OpenAPITool component to invoke **any** OpenAPI service/tool by providing the OpenAPI - specification and credentials. - """ - - def __init__( - self, - generator_api: LLMProvider, - generator_api_params: Optional[Dict[str, Any]] = None, - spec: Optional[Union[str, Path]] = None, - credentials: Optional[Secret] = None, - allowed_operations: Optional[List[str]] = None, - ): # pylint: disable=too-many-positional-arguments - """ - Initialize the OpenAPITool component. - - :param generator_api: The API provider for the chat generator. - :param generator_api_params: Parameters to pass for the chat generator creation. - :param spec: OpenAPI specification for the tool/service. This can be a URL, a local file path, or - an OpenAPI service specification provided as a string. - :param credentials: Credentials for the tool/service. - :param allowed_operations: A list of operations to register with LLMs via the LLM tools parameter. Use - operationId field in the OpenAPI spec path/operation to specify the operation names to use. If not specified, - all operations found in the OpenAPI spec will be registered with LLMs. - """ - self.generator_api = generator_api - self.generator_api_params = generator_api_params or {} # store the generator API parameters for serialization - self.chat_generator = self._init_generator(generator_api, generator_api_params or {}) - self.config_openapi: Optional[ClientConfiguration] = None - self.open_api_service: Optional[OpenAPIServiceClient] = None - self.spec = spec # store the spec for serialization - self.credentials = credentials # store the credentials for serialization - self.allowed_operations = allowed_operations - if spec: - if os.path.isfile(spec): - openapi_spec = OpenAPISpecification.from_file(spec) - elif is_valid_http_url(str(spec)): - openapi_spec = OpenAPISpecification.from_url(str(spec)) - else: - raise ValueError(f"Invalid OpenAPI specification source {spec}. Expected valid file path or URL") - self.config_openapi = ClientConfiguration( - openapi_spec=openapi_spec, - credentials=credentials.resolve_value() if credentials else None, - llm_provider=generator_api, - operations_filter=(lambda f: f["operationId"] in allowed_operations) if allowed_operations else None, - ) - self.open_api_service = OpenAPIServiceClient(self.config_openapi) - - @component.output_types(service_response=List[ChatMessage]) - def run( - self, - messages: List[ChatMessage], - fc_generator_kwargs: Optional[Dict[str, Any]] = None, - spec: Optional[Union[str, Path]] = None, - credentials: Optional[Secret] = None, - ) -> Dict[str, List[ChatMessage]]: - """ - Invokes the underlying OpenAPI service/tool with the function calling payload generated by the chat generator. - - :param messages: List of ChatMessages to generate function calling payload (e.g. human instructions). The last - message should be human instruction containing enough information to generate the function calling payload - suitable for the OpenAPI service/tool used. See the examples in the class docstring. - :param fc_generator_kwargs: Additional arguments for the function calling payload generation process. - :param spec: OpenAPI specification for the tool/service, overrides the one provided at initialization. - :param credentials: Credentials for the tool/service, overrides the one provided at initialization. - :returns: a dictionary containing the service response with the following key: - - `service_response`: List of ChatMessages containing the service response. ChatMessages are generated - based on the response from the OpenAPI service/tool and contains the JSON response from the service. - If there is an error during the invocation, the response will be a ChatMessage with the error message under - the `error` key. - """ - last_message = messages[-1] - if not last_message.is_from(ChatRole.USER): - raise ValueError(f"{last_message} not from the user") - if not last_message.content: - raise ValueError("Function calling instruction message content is empty.") - - # build a new ClientConfiguration and OpenAPIServiceClient if a runtime tool_spec is provided - openapi_service: Optional[OpenAPIServiceClient] = self.open_api_service - config_openapi: Optional[ClientConfiguration] = self.config_openapi - if spec: - if os.path.isfile(spec): - openapi_spec = OpenAPISpecification.from_file(spec) - elif is_valid_http_url(str(spec)): - openapi_spec = OpenAPISpecification.from_url(str(spec)) - else: - raise ValueError(f"Invalid OpenAPI specification source {spec}. Expected valid file path or URL") - - config_openapi = ClientConfiguration( - openapi_spec=openapi_spec, - credentials=credentials.resolve_value() if credentials else None, - llm_provider=self.generator_api, - ) - openapi_service = OpenAPIServiceClient(config_openapi) - - if not openapi_service or not config_openapi: - raise ValueError( - "OpenAPI specification not provided. Please provide an OpenAPI specification either at initialization " - "or during runtime." - ) - - # merge fc_generator_kwargs, tools definitions comes from the OpenAPI spec, other kwargs are passed by the user - fc_generator_kwargs = { - "tools": config_openapi.get_tools_definitions(), - **(fc_generator_kwargs or {}), - } - - # generate function calling payload with the chat generator - logger.debug( - "Invoking chat generator with {message} to generate function calling payload.", - message=last_message.content, - ) - fc_payload = self.chat_generator.run(messages, generation_kwargs=fc_generator_kwargs) - try: - invocation_payload = json.loads(fc_payload["replies"][0].content) - logger.debug("Invoking tool with {payload}", payload=invocation_payload) - service_response = openapi_service.invoke(invocation_payload) - except Exception as e: # pylint: disable=broad-exception-caught - logger.error("Error invoking OpenAPI endpoint. Error: {e}", e=str(e)) - service_response = {"error": str(e)} - # We disable ensure_ascii so special chars like emojis are not converted - response_messages = [ChatMessage.from_user(json.dumps(service_response, ensure_ascii=False))] - - return {"service_response": response_messages} - - def to_dict(self) -> Dict[str, Any]: - """ - Serialize this component to a dictionary. - - :returns: - The serialized component as a dictionary. - """ - serialize_secrets_inplace(self.generator_api_params, keys=["api_key"], recursive=True) - return default_to_dict( - self, - generator_api=self.generator_api.value, - generator_api_params=self.generator_api_params, - spec=self.spec, - credentials=self.credentials.to_dict() if self.credentials else None, - allowed_operations=self.allowed_operations, - ) - - @classmethod - def from_dict(cls, data: Dict[str, Any]) -> "OpenAPITool": - """ - Deserialize this component from a dictionary. - - :param data: The dictionary representation of this component. - :returns: - The deserialized component instance. - """ - deserialize_secrets_inplace(data["init_parameters"], keys=["credentials"]) - deserialize_secrets_inplace(data["init_parameters"]["generator_api_params"], keys=["api_key"]) - init_params = data.get("init_parameters", {}) - generator_api = init_params.get("generator_api") - data["init_parameters"]["generator_api"] = LLMProvider.from_str(generator_api) - return default_from_dict(cls, data) - - def _init_generator(self, generator_api: LLMProvider, generator_api_params: Dict[str, Any]): - """ - Initialize the chat generator based on the specified API provider and parameters. - """ - if generator_api == LLMProvider.OPENAI: - return OpenAIChatGenerator(**generator_api_params) - if generator_api == LLMProvider.COHERE: - cohere_import.check() - return CohereChatGenerator(**generator_api_params) - if generator_api == LLMProvider.ANTHROPIC: - anthropic_import.check() - return AnthropicChatGenerator(**generator_api_params) - raise ValueError(f"Unsupported generator API: {generator_api}") diff --git a/haystack_experimental/components/tools/openapi/types.py b/haystack_experimental/components/tools/openapi/types.py deleted file mode 100644 index 4216ced0..00000000 --- a/haystack_experimental/components/tools/openapi/types.py +++ /dev/null @@ -1,252 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 - -import json -from dataclasses import dataclass, field -from enum import Enum -from pathlib import Path -from typing import Any, Dict, List, Literal, Optional, Union - -import requests -import yaml -from haystack.lazy_imports import LazyImport - -from haystack_experimental.components.tools.utils import normalize_function_name - -with LazyImport("Run 'pip install jsonref'") as jsonref_import: - # pylint: disable=import-error - import jsonref - - -VALID_HTTP_METHODS = [ - "get", - "put", - "post", - "delete", - "options", - "head", - "patch", - "trace", -] - - -def path_to_operation_id(path: str, http_method: str = "get") -> str: - """ - Converts an OpenAPI spec path to synthetic operationId. - - :param path: The OpenAPI spec path to convert. - :param http_method: The HTTP method to use for the operationId. - :returns: The operationId. - """ - return normalize_function_name(path + "_" + http_method.lower()) - - -class LLMProvider(Enum): - """ - LLM providers supported by `OpenAPITool`. - """ - - OPENAI = "openai" - ANTHROPIC = "anthropic" - COHERE = "cohere" - - @staticmethod - def from_str(string: str) -> "LLMProvider": - """ - Convert a string to a LLMProvider enum. - """ - provider_map = {e.value: e for e in LLMProvider} - provider = provider_map.get(string) - if provider is None: - msg = ( - f"Invalid LLMProvider '{string}'" - f"Supported LLMProviders are: {list(provider_map.keys())}" - ) - raise ValueError(msg) - return provider - - -@dataclass -class Operation: - """ - Represents an operation in an OpenAPI specification - - See https://spec.openapis.org/oas/latest.html#paths-object for details. - Path objects can contain multiple operations, each with a unique combination of path and method. - - :param path: Path of the operation. - :param method: HTTP method of the operation. - :param operation_dict: Operation details from OpenAPI spec - :param spec_dict: The encompassing OpenAPI specification. - :param security_requirements: A list of security requirements for the operation. - :param request_body: Request body details. - :param parameters: Parameters for the operation. - """ - - path: str - method: str - operation_dict: Dict[str, Any] - spec_dict: Dict[str, Any] - security_requirements: List[Dict[str, List[str]]] = field(init=False) - request_body: Dict[str, Any] = field(init=False) - parameters: List[Dict[str, Any]] = field(init=False) - - def __post_init__(self): - if self.method.lower() not in VALID_HTTP_METHODS: - raise ValueError(f"Invalid HTTP method: {self.method}") - self.method = self.method.lower() - self.security_requirements = self.operation_dict.get( - "security", [] - ) or self.spec_dict.get("security", []) - self.request_body = self.operation_dict.get("requestBody", {}) - self.parameters = self.operation_dict.get( - "parameters", [] - ) + self.spec_dict.get("paths", {}).get(self.path, {}).get("parameters", []) - - def get_parameters( - self, location: Optional[Literal["header", "query", "path"]] = None - ) -> List[Dict[str, Any]]: - """ - Get the parameters for the operation. - - :param location: The location of the parameters to get. - :returns: The parameters for the operation as a list of dictionaries. - """ - if location: - return [param for param in self.parameters if param["in"] == location] - return self.parameters - - def get_server(self, server_index: int = 0) -> str: - """ - Get the servers for the operation. - - :param server_index: The index of the server to use. - :returns: The server URL. - :raises ValueError: If no servers are found in the specification. - """ - # servers can be defined at the operation level, path level, or at the root level - # search for servers in the following order: operation, path, root - servers = ( - self.operation_dict.get("servers", []) - or self.spec_dict.get("paths", {}).get(self.path, {}).get("servers", []) - or self.spec_dict.get("servers", []) - ) - if not servers: - raise ValueError("No servers found in the provided specification.") - if not 0 <= server_index < len(servers): - raise ValueError( - f"Server index {server_index} is out of bounds. " - f"Only {len(servers)} servers found." - ) - return servers[server_index].get( - "url" - ) # just use the first server from the list - - -class OpenAPISpecification: - """ - Represents an OpenAPI specification. See https://spec.openapis.org/oas/latest.html for details. - """ - - def __init__(self, spec_dict: Dict[str, Any]): - """ - Initialize an OpenAPISpecification instance. - - :param spec_dict: The OpenAPI specification as a dictionary. - """ - jsonref_import.check() - if not isinstance(spec_dict, Dict): - raise ValueError( - f"Invalid OpenAPI specification, expected a dictionary: {spec_dict}" - ) - # just a crude sanity check, by no means a full validation - if "openapi" not in spec_dict or "paths" not in spec_dict: - raise ValueError( - "Invalid OpenAPI specification format. See https://swagger.io/specification/ for details.", - spec_dict, - ) - self.spec_dict = jsonref.replace_refs(spec_dict) - - @classmethod - def from_str(cls, content: str) -> "OpenAPISpecification": - """ - Create an OpenAPISpecification instance from a string. - - :param content: The string content of the OpenAPI specification. - :returns: The OpenAPISpecification instance. - :raises ValueError: If the content cannot be decoded as JSON or YAML. - """ - try: - loaded_spec = json.loads(content) - except json.JSONDecodeError: - try: - loaded_spec = yaml.safe_load(content) - except yaml.YAMLError as e: - raise ValueError( - "Content cannot be decoded as JSON or YAML: " + str(e) - ) from e - return cls(loaded_spec) - - @classmethod - def from_file(cls, spec_file: Union[str, Path]) -> "OpenAPISpecification": - """ - Create an OpenAPISpecification instance from a file. - - :param spec_file: The file path to the OpenAPI specification. - :returns: The OpenAPISpecification instance. - :raises FileNotFoundError: If the specified file does not exist. - :raises IOError: If an I/O error occurs while reading the file. - :raises ValueError: If the file content cannot be decoded as JSON or YAML. - """ - with open(spec_file, encoding="utf-8") as file: - content = file.read() - return cls.from_str(content) - - @classmethod - def from_url(cls, url: str) -> "OpenAPISpecification": - """ - Create an OpenAPISpecification instance from a URL. - - :param url: The URL to fetch the OpenAPI specification from. - :returns: The OpenAPISpecification instance. - :raises ConnectionError: If fetching the specification from the URL fails. - """ - try: - response = requests.get(url, timeout=10) - response.raise_for_status() - content = response.text - except requests.RequestException as e: - raise ConnectionError( - f"Failed to fetch the specification from URL: {url}. {e!s}" - ) from e - return cls.from_str(content) - - def find_operation_by_id(self, op_id: str) -> Operation: - """ - Find an Operation by operationId. - - :param op_id: The operationId of the operation. - :returns: The matching operation - :raises ValueError: If no operation is found with the given operationId. - """ - for path, path_value in self.spec_dict.get("paths", {}).items(): - operations = { - method: operation_dict - for method, operation_dict in path_value.items() - if method.lower() in VALID_HTTP_METHODS - } - - for method, operation_dict in operations.items(): - operation_id = operation_dict.get("operationId", path_to_operation_id(path, method)) - if normalize_function_name(operation_id) == op_id: - return Operation(path, method, operation_dict, self.spec_dict) - raise ValueError(f"No operation found with operationId {op_id}") - - def get_security_schemes(self) -> Dict[str, Dict[str, Any]]: - """ - Get the security schemes from the OpenAPI specification. - - :returns: The security schemes as a dictionary. - """ - return self.spec_dict.get("components", {}).get("securitySchemes", {}) diff --git a/haystack_experimental/components/tools/utils.py b/haystack_experimental/components/tools/utils.py deleted file mode 100644 index 64da2114..00000000 --- a/haystack_experimental/components/tools/utils.py +++ /dev/null @@ -1,57 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 - -import re -from typing import Any, Dict - - -def normalize_tool_definition(data: Dict[str, Any]) -> Dict[str, Any]: - """ - Normalizes the given tool definition by adjusting its properties to LLM requirements. - - While various LLMs have slightly different requirements for tool definitions, we normalize them to a common - format that is compatible with OpenAI, Anthropic, and Cohere LLMs: - - tool names have to match the pattern ^[a-zA-Z0-9_]+$ and are truncated to 64 characters - - tool/parameter descriptions are truncated to 1024 characters - - For reference on tool definition formats, see: - - https://cookbook.openai.com/examples/how_to_call_functions_with_chat_models#basic-concepts - - https://docs.anthropic.com/en/docs/build-with-claude/tool-use - - https://docs.cohere.com/docs/tool-use - - :param data: The function calling definition(s) to normalize. - :returns: A normalized function calling definition. - """ - normalized_data: Dict[str, Any] = {} - for key, value in data.items(): - # all LLMs tool definitions have tool (function) name and description on the same level - # if we find it then normalize the function name - if key == "name" and "description" in data.keys(): - normalized_data[key] = normalize_function_name(value) - elif key == "description": - normalized_data[key] = value[:1024] - elif isinstance(value, dict): - # recursively normalize nested descriptions (e.g. tool parameters) - normalized_data[key] = normalize_tool_definition(value) - else: - normalized_data[key] = value - return normalized_data - - -def normalize_function_name(name: str) -> str: - """ - Normalizes the function name to match the LLM function naming requirements. - - While various LLMs have slightly different requirements for tool (function) names, we normalize them to - a common format that is compatible with OpenAI, Anthropic, and Cohere LLMs: - - The function name must match the pattern ^[a-zA-Z0-9_]+$ - - The function name must be truncated to 64 characters - - :param name: The original function name. - :returns: A normalized function name that matches the allowed pattern. - """ - # Replace characters not allowed in the pattern with underscores - normalized = re.sub(r"[^a-zA-Z0-9_]+", "_", name) - # Remove leading and trailing underscores and truncate to 64 characters - return normalized.strip("_")[:64] diff --git a/test/components/tools/openai/__init__.py b/test/components/tools/openai/__init__.py deleted file mode 100644 index 3f4ac9d8..00000000 --- a/test/components/tools/openai/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 \ No newline at end of file diff --git a/test/components/tools/openai/test_function_caller.py b/test/components/tools/openai/test_function_caller.py deleted file mode 100644 index 9182120f..00000000 --- a/test/components/tools/openai/test_function_caller.py +++ /dev/null @@ -1,69 +0,0 @@ -# SPDX-FileCopyrightText: 2023-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 -import os -import json -import pytest - -# from haystack.utils import Secret -from haystack_experimental.components.tools import OpenAIFunctionCaller -from haystack.dataclasses import ChatMessage - -WEATHER_INFO = { - "Berlin": {"weather": "mostly sunny", "temperature": 7, "unit": "celsius"}, - "Paris": {"weather": "mostly cloudy", "temperature": 8, "unit": "celsius"}, - "Rome": {"weather": "sunny", "temperature": 14, "unit": "celsius"}, - "Madrid": {"weather": "sunny", "temperature": 10, "unit": "celsius"}, - "London": {"weather": "cloudy", "temperature": 9, "unit": "celsius"}, -} - - -def mock_weather_func(location): - if location in WEATHER_INFO: - return WEATHER_INFO[location] - else: - return {"weather": "sunny", "temperature": 21.8, "unit": "fahrenheit"} - -class TestOpenAIFunctionCaller: - - def test_init(self, monkeypatch): - component = OpenAIFunctionCaller(available_functions = {"mock_weather_func": mock_weather_func}) - assert component.available_functions == {"mock_weather_func": mock_weather_func} - - def test_successful_function_call(self, monkeypatch): - component = OpenAIFunctionCaller(available_functions = {"mock_weather_func": mock_weather_func}) - mock_assistant_message = ChatMessage.from_assistant(content='[{"id": "mock-id", "function": {"arguments": "{\\"location\\":\\"Berlin\\"}", "name": "mock_weather_func"}, "type": "function"}]', - meta={"finish_reason": "tool_calls"}) - result = component.run(messages=[mock_assistant_message]) - result_obj = json.loads(result["function_replies"][-1].content) - assert result_obj['weather'] == WEATHER_INFO['Berlin']['weather'] - assert result_obj['temperature'] == WEATHER_INFO['Berlin']['temperature'] - assert result_obj['unit'] == WEATHER_INFO['Berlin']['unit'] - - - def test_failing_function_call(self, monkeypatch): - component = OpenAIFunctionCaller(available_functions = {"mock_weather_func": mock_weather_func}) - mock_assistant_message = ChatMessage.from_assistant(content='[{"id": "mock-id", "function": {"arguments": "{\\"location\\":\\"Berlin\\"}", "name": "mock_weather"}, "type": "function"}]', - meta={"finish_reason": "tool_calls"}) - result = component.run(messages=[mock_assistant_message]) - assert result["function_replies"][-1].content == "I'm sorry, I tried to run a function that did not exist. Would you like me to correct it and try again?" - - def test_to_dict(self, monkeypatch): - component = OpenAIFunctionCaller(available_functions = {"mock_weather_func": mock_weather_func}) - data = component.to_dict() - assert data == { - "type": "haystack_experimental.components.tools.openai.function_caller.OpenAIFunctionCaller", - "init_parameters": { - "available_functions": {'mock_weather_func': 'test.components.tools.openai.test_function_caller.mock_weather_func'} - }, - } - - def test_from_dict(self, monkeypatch): - data = { - "type": "haystack_experimental.components.tools.openai.function_caller.OpenAIFunctionCaller", - "init_parameters": { - "available_functions": {'mock_weather_func': 'test.components.tools.openai.test_function_caller.mock_weather_func'}, - }, - } - component: OpenAIFunctionCaller = OpenAIFunctionCaller.from_dict(data) - assert component.available_functions == {'mock_weather_func': mock_weather_func} \ No newline at end of file diff --git a/test/components/tools/openapi/__init__.py b/test/components/tools/openapi/__init__.py deleted file mode 100644 index c1764a6e..00000000 --- a/test/components/tools/openapi/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 diff --git a/test/components/tools/openapi/conftest.py b/test/components/tools/openapi/conftest.py deleted file mode 100644 index eaa7e7d2..00000000 --- a/test/components/tools/openapi/conftest.py +++ /dev/null @@ -1,79 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 -import os -from pathlib import Path -from typing import Union -from urllib.parse import urlparse - -import pytest -from fastapi import FastAPI -from fastapi.testclient import TestClient -from haystack.utils.url_validation import is_valid_http_url - -from haystack_experimental.components.tools.openapi._openapi import HttpClientError -from haystack_experimental.components.tools.openapi.types import OpenAPISpecification - - -@pytest.fixture() -def test_files_path(): - return Path(__file__).parent.parent.parent.parent / "test_files" - - -def create_openapi_spec(openapi_spec: Union[Path, str]) -> OpenAPISpecification: - if isinstance(openapi_spec, (str, Path)) and os.path.isfile(openapi_spec): - return OpenAPISpecification.from_file(openapi_spec) - elif isinstance(openapi_spec, str): - if is_valid_http_url(openapi_spec): - return OpenAPISpecification.from_url(openapi_spec) - else: - return OpenAPISpecification.from_str(openapi_spec) - else: - raise ValueError( - "Invalid OpenAPI specification format. Expected file path or dictionary." - ) - -def env_var_set(var_name): - return var_name in os.environ and os.environ[var_name].strip() - -def provider_api_key_set(provider): - if provider == "openai": - return env_var_set("OPENAI_API_KEY") - elif provider == "anthropic": - return env_var_set("ANTHROPIC_API_KEY") - elif provider == "cohere": - return env_var_set("COHERE_API_KEY") - return False - -class FastAPITestClient: - - def __init__(self, app: FastAPI): - self.app = app - self.client = TestClient(app) - - def strip_host(self, url: str) -> str: - parsed_url = urlparse(url) - new_path = parsed_url.path - if parsed_url.query: - new_path += "?" + parsed_url.query - return new_path - - def __call__(self, request: dict) -> dict: - # OAS spec will list a server URL, but FastAPI doesn't need it for local testing, in fact it will fail - # if the URL has a host. So we strip it here. - url = self.strip_host(request["url"]) - try: - response = self.client.request( - request["method"], - url, - headers=request.get("headers", {}), - params=request.get("params", {}), - json=request.get("json", None), - auth=request.get("auth", None), - cookies=request.get("cookies", {}), - ) - response.raise_for_status() - return response.json() - except Exception as e: - # Handle HTTP errors - raise HttpClientError(f"HTTP error occurred: {e}") from e diff --git a/test/components/tools/openapi/test_openapi_client.py b/test/components/tools/openapi/test_openapi_client.py deleted file mode 100644 index ad745125..00000000 --- a/test/components/tools/openapi/test_openapi_client.py +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 - - -from fastapi import FastAPI -from fastapi.responses import JSONResponse -from pydantic import BaseModel - -from haystack_experimental.components.tools.openapi._openapi import OpenAPIServiceClient, ClientConfiguration -from test.components.tools.openapi.conftest import FastAPITestClient, create_openapi_spec - -""" -Tests OpenAPIServiceClient with three FastAPI apps for different parameter types: - -- **greet_mix_params_body**: A POST endpoint `/greet/` accepting a JSON payload with a message, returning a -greeting with the name from the URL and the message from the payload. - -- **greet_params_only**: A GET endpoint `/greet-params/` taking a URL parameter, returning a greeting with -the name from the URL. - -- **greet_request_body_only**: A POST endpoint `/greet-body` accepting a JSON payload with a name and message, -returning a greeting with both. - -OpenAPI specs for these endpoints are in `openapi_greeting_service.yml` in `test/test_files` directory. -""" - - -class GreetBody(BaseModel): - message: str - name: str - - -class MessageBody(BaseModel): - message: str - - -# FastAPI app definitions -def create_greet_mix_params_body_app() -> FastAPI: - app = FastAPI() - - @app.post("/greet/{name}") - def greet(name: str, body: MessageBody): - greeting = f"{body.message}, {name} from mix_params_body!" - return JSONResponse(content={"greeting": greeting}) - - return app - - -def create_greet_params_only_app() -> FastAPI: - app = FastAPI() - - @app.get("/greet-params/{name}") - def greet_params(name: str): - greeting = f"Hello, {name} from params_only!" - return JSONResponse(content={"greeting": greeting}) - - return app - - -def create_greet_request_body_only_app() -> FastAPI: - app = FastAPI() - - @app.post("/greet-body") - def greet_request_body(body: GreetBody): - greeting = f"{body.message}, {body.name} from request_body_only!" - return JSONResponse(content={"greeting": greeting}) - - return app - - -class TestOpenAPI: - - def test_greet_mix_params_body(self, test_files_path): - config = ClientConfiguration(openapi_spec=create_openapi_spec(test_files_path / "yaml" / "openapi_greeting_service.yml"), - request_sender=FastAPITestClient(create_greet_mix_params_body_app())) - client = OpenAPIServiceClient(config) - payload = { - "id": "call_NJr1NBz2Th7iUWJpRIJZoJIA", - "function": { - "arguments": '{"name": "John", "message": "Bonjour"}', - "name": "greet", - }, - "type": "function", - } - response = client.invoke(payload) - assert response == {"greeting": "Bonjour, John from mix_params_body!"} - - def test_greet_params_only(self, test_files_path): - config = ClientConfiguration(openapi_spec=create_openapi_spec(test_files_path / "yaml" / "openapi_greeting_service.yml"), - request_sender=FastAPITestClient(create_greet_params_only_app())) - client = OpenAPIServiceClient(config) - payload = { - "id": "call_NJr1NBz2Th7iUWJpRIJZoJIA", - "function": { - "arguments": '{"name": "John"}', - "name": "greetParams", - }, - "type": "function", - } - response = client.invoke(payload) - assert response == {"greeting": "Hello, John from params_only!"} - - def test_greet_request_body_only(self, test_files_path): - config = ClientConfiguration(openapi_spec=create_openapi_spec(test_files_path / "yaml" / "openapi_greeting_service.yml"), - request_sender=FastAPITestClient(create_greet_request_body_only_app())) - client = OpenAPIServiceClient(config) - payload = { - "id": "call_NJr1NBz2Th7iUWJpRIJZoJIA", - "function": { - "arguments": '{"name": "John", "message": "Hola"}', - "name": "greetBody", - }, - "type": "function", - } - response = client.invoke(payload) - assert response == {"greeting": "Hola, John from request_body_only!"} diff --git a/test/components/tools/openapi/test_openapi_client_auth.py b/test/components/tools/openapi/test_openapi_client_auth.py deleted file mode 100644 index 6a91bcdd..00000000 --- a/test/components/tools/openapi/test_openapi_client_auth.py +++ /dev/null @@ -1,187 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 - - -from fastapi import Depends, FastAPI, HTTPException, status -from fastapi.responses import JSONResponse -from fastapi.security import ( - APIKeyCookie, - APIKeyHeader, - APIKeyQuery, - HTTPAuthorizationCredentials, - HTTPBasic, - HTTPBasicCredentials, - HTTPBearer, -) - -from haystack_experimental.components.tools.openapi._openapi import OpenAPIServiceClient, ClientConfiguration -from test.components.tools.openapi.conftest import FastAPITestClient, create_openapi_spec - -API_KEY = "secret_api_key" -BASIC_AUTH_USERNAME = "admin" -BASIC_AUTH_PASSWORD = "secret_password" - -API_KEY_QUERY = "secret_api_key_query" -API_KEY_COOKIE = "secret_api_key_cookie" -BEARER_TOKEN = "secret_bearer_token" - -OAUTH_TOKEN = "secret-oauth-token" - -api_key_query = APIKeyQuery(name="api_key") -api_key_cookie = APIKeyCookie(name="api_key") -bearer_auth = HTTPBearer() - -api_key_header = APIKeyHeader(name="X-API-Key") -basic_auth_http = HTTPBasic() - - -def create_greet_api_key_query_app() -> FastAPI: - app = FastAPI() - - def api_key_query_auth(api_key: str = Depends(api_key_query)): - if api_key != API_KEY_QUERY: - raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid API key") - return api_key - - @app.get("/greet-api-key-query/{name}") - def greet_api_key_query(name: str, api_key: str = Depends(api_key_query_auth)): - greeting = f"Hello, {name} from api_key_query_auth, using {api_key}" - return JSONResponse(content={"greeting": greeting}) - - return app - - -def create_greet_api_key_cookie_app() -> FastAPI: - app = FastAPI() - - def api_key_cookie_auth(api_key: str = Depends(api_key_cookie)): - if api_key != API_KEY_COOKIE: - raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid API key") - return api_key - - @app.get("/greet-api-key-cookie/{name}") - def greet_api_key_cookie(name: str, api_key: str = Depends(api_key_cookie_auth)): - greeting = f"Hello, {name} from api_key_cookie_auth, using {api_key}" - return JSONResponse(content={"greeting": greeting}) - - return app - - -def create_greet_bearer_auth_app() -> FastAPI: - app = FastAPI() - - def bearer_auth_scheme( - credentials: HTTPAuthorizationCredentials = Depends(bearer_auth), # noqa: B008 - ): - if credentials.scheme != "Bearer" or credentials.credentials != BEARER_TOKEN: - raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid token") - return credentials.credentials - - @app.get("/greet-bearer-auth/{name}") - def greet_bearer_auth(name: str, token: str = Depends(bearer_auth_scheme)): - greeting = f"Hello, {name} from bearer_auth, using {token}" - return JSONResponse(content={"greeting": greeting}) - - return app - - -def create_greet_api_key_auth_app() -> FastAPI: - app = FastAPI() - - def api_key_auth(api_key: str = Depends(api_key_header)): - if api_key != API_KEY: - raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid API key") - return api_key - - @app.get("/greet-api-key/{name}") - def greet_api_key(name: str, api_key: str = Depends(api_key_auth)): - greeting = f"Hello, {name} from api_key_auth, using {api_key}" - return JSONResponse(content={"greeting": greeting}) - - return app - - -def create_greet_basic_auth_app() -> FastAPI: - app = FastAPI() - - def basic_auth(credentials: HTTPBasicCredentials = Depends(basic_auth_http)): # noqa: B008 - if credentials.username != BASIC_AUTH_USERNAME or credentials.password != BASIC_AUTH_PASSWORD: - raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid credentials") - return credentials.username - - @app.get("/greet-basic-auth/{name}") - def greet_basic_auth(name: str, username: str = Depends(basic_auth)): - greeting = f"Hello, {name} from basic_auth, using {username}" - return JSONResponse(content={"greeting": greeting}) - - return app - - -def create_greet_oauth_auth_app() -> FastAPI: - app = FastAPI() - - def oauth_auth(token: HTTPAuthorizationCredentials = Depends(HTTPBearer())): # noqa: B008 - if token.credentials != OAUTH_TOKEN: - raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid token") - return token - - @app.get("/greet-oauth/{name}") - def greet_oauth(name: str, token: HTTPAuthorizationCredentials = Depends(oauth_auth)): # noqa: B008 - greeting = f"Hello, {name} from oauth_auth, using {token}" - return JSONResponse(content={"greeting": greeting}) - - return app - - -class TestOpenAPIAuth: - - def test_greet_api_key_auth(self, test_files_path): - config = ClientConfiguration(openapi_spec=create_openapi_spec(test_files_path / "yaml" / "openapi_greeting_service.yml"), - request_sender=FastAPITestClient(create_greet_api_key_auth_app()), - credentials=API_KEY) - client = OpenAPIServiceClient(config) - payload = { - "id": "call_NJr1NBz2Th7iUWJpRIJZoJIA", - "function": { - "arguments": '{"name": "John"}', - "name": "greetApiKey", - }, - "type": "function", - } - response = client.invoke(payload) - assert response == {"greeting": "Hello, John from api_key_auth, using secret_api_key"} - - def test_greet_api_key_query_auth(self, test_files_path): - config = ClientConfiguration(openapi_spec=create_openapi_spec(test_files_path / "yaml" / "openapi_greeting_service.yml"), - request_sender=FastAPITestClient(create_greet_api_key_query_app()), - credentials=API_KEY_QUERY) - client = OpenAPIServiceClient(config) - payload = { - "id": "call_NJr1NBz2Th7iUWJpRIJZoJIA", - "function": { - "arguments": '{"name": "John"}', - "name": "greetApiKeyQuery", - }, - "type": "function", - } - response = client.invoke(payload) - assert response == {"greeting": "Hello, John from api_key_query_auth, using secret_api_key_query"} - - def test_greet_api_key_cookie_auth(self, test_files_path): - - config = ClientConfiguration(openapi_spec=create_openapi_spec(test_files_path / "yaml" / "openapi_greeting_service.yml"), - request_sender=FastAPITestClient(create_greet_api_key_cookie_app()), - credentials=API_KEY_COOKIE) - - client = OpenAPIServiceClient(config) - payload = { - "id": "call_NJr1NBz2Th7iUWJpRIJZoJIA", - "function": { - "arguments": '{"name": "John"}', - "name": "greetApiKeyCookie", - }, - "type": "function", - } - response = client.invoke(payload) - assert response == {"greeting": "Hello, John from api_key_cookie_auth, using secret_api_key_cookie"} \ No newline at end of file diff --git a/test/components/tools/openapi/test_openapi_client_complex_request_body.py b/test/components/tools/openapi/test_openapi_client_complex_request_body.py deleted file mode 100644 index 8ce2273e..00000000 --- a/test/components/tools/openapi/test_openapi_client_complex_request_body.py +++ /dev/null @@ -1,85 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 - - -import json -from typing import List - -import pytest -from fastapi import FastAPI -from fastapi.responses import JSONResponse -from pydantic import BaseModel - -from haystack_experimental.components.tools.openapi._openapi import OpenAPIServiceClient, ClientConfiguration -from test.components.tools.openapi.conftest import FastAPITestClient, create_openapi_spec - - -class Customer(BaseModel): - name: str - email: str - - -class OrderItem(BaseModel): - product: str - quantity: int - - -class Order(BaseModel): - customer: Customer - items: List[OrderItem] - - -class OrderResponse(BaseModel): - orderId: str # noqa: N815 - status: str - totalAmount: float # noqa: N815 - - -def create_order_app() -> FastAPI: - app = FastAPI() - - @app.post("/orders") - def create_order(order: Order): - total_amount = sum(item.quantity * 10 for item in order.items) - response = OrderResponse( - orderId="ORDER-001", - status="CREATED", - totalAmount=total_amount, - ) - return JSONResponse(content=response.model_dump(), status_code=201) - - return app - - -class TestComplexRequestBody: - - @pytest.mark.parametrize("spec_file_path", ["openapi_order_service.yml", "openapi_order_service.json"]) - def test_create_order(self, spec_file_path, test_files_path): - path_element = "yaml" if spec_file_path.endswith(".yml") else "json" - - config = ClientConfiguration(openapi_spec=create_openapi_spec(test_files_path / path_element / spec_file_path), - request_sender=FastAPITestClient(create_order_app())) - - client = OpenAPIServiceClient(config) - order_json = { - "customer": {"name": "John Doe", "email": "john@example.com"}, - "items": [ - {"product": "Product A", "quantity": 2}, - {"product": "Product B", "quantity": 1}, - ], - } - payload = { - "id": "call_NJr1NBz2Th7iUWJpRIJZoJIA", - "function": { - "arguments": json.dumps(order_json), - "name": "createOrder", - }, - "type": "function", - } - response = client.invoke(payload) - assert response == { - "orderId": "ORDER-001", - "status": "CREATED", - "totalAmount": 30, - } diff --git a/test/components/tools/openapi/test_openapi_client_complex_request_body_mixed.py b/test/components/tools/openapi/test_openapi_client_complex_request_body_mixed.py deleted file mode 100644 index 9624f7d7..00000000 --- a/test/components/tools/openapi/test_openapi_client_complex_request_body_mixed.py +++ /dev/null @@ -1,86 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 - - -import json - -from fastapi import FastAPI -from fastapi.responses import JSONResponse -from pydantic import BaseModel - -from haystack_experimental.components.tools.openapi._openapi import OpenAPIServiceClient, ClientConfiguration -from test.components.tools.openapi.conftest import FastAPITestClient, create_openapi_spec - - -class Identification(BaseModel): - type: str - number: str - - -class Payer(BaseModel): - name: str - email: str - identification: Identification - - -class PaymentRequest(BaseModel): - transaction_amount: float - description: str - payment_method_id: str - payer: Payer - - -class PaymentResponse(BaseModel): - transaction_id: str - status: str - message: str - - -def create_payment_app() -> FastAPI: - app = FastAPI() - - @app.post("/new_payment") - def process_payment(payment: PaymentRequest): - # sanity - assert payment.transaction_amount == 100.0 - response = PaymentResponse( - transaction_id="TRANS-12345", status="SUCCESS", message="Payment processed successfully." - ) - return JSONResponse(content=response.model_dump(), status_code=200) - - return app - - -# Write the unit test -class TestPaymentProcess: - - def test_process_payment(self, test_files_path): - config = ClientConfiguration(openapi_spec=create_openapi_spec(test_files_path / "json" / "complex_types_openapi_service.json"), - request_sender=FastAPITestClient(create_payment_app())) - client = OpenAPIServiceClient(config) - - payment_json = { - "transaction_amount": 100.0, - "description": "Test Payment", - "payment_method_id": "CARD-123", - "payer": { - "name": "Alice Smith", - "email": "alice@example.com", - "identification": {"type": "CPF", "number": "123.456.789-00"}, - }, - } - payload = { - "id": "call_uniqueID123", - "function": { - "arguments": json.dumps(payment_json), - "name": "processPayment", - }, - "type": "function", - } - response = client.invoke(payload) - assert response == { - "transaction_id": "TRANS-12345", - "status": "SUCCESS", - "message": "Payment processed successfully.", - } diff --git a/test/components/tools/openapi/test_openapi_client_edge_cases.py b/test/components/tools/openapi/test_openapi_client_edge_cases.py deleted file mode 100644 index 2dd0a9b1..00000000 --- a/test/components/tools/openapi/test_openapi_client_edge_cases.py +++ /dev/null @@ -1,54 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 - - -import pytest - -from haystack_experimental.components.tools.openapi._openapi import ClientConfiguration, OpenAPIServiceClient -from test.components.tools.openapi.conftest import FastAPITestClient, create_openapi_spec - - -class TestEdgeCases: - - def test_missing_operation_id(self, test_files_path): - config = ClientConfiguration(openapi_spec=create_openapi_spec(test_files_path / "yaml" / "openapi_edge_cases.yml"), - request_sender=FastAPITestClient(None)) - client = OpenAPIServiceClient(config) - - payload = { - "type": "function", - "function": { - "arguments": '{"name": "John", "message": "Hola"}', - "name": "missingOperationId", - }, - } - with pytest.raises(ValueError, match="No operation found with operationId"): - client.invoke(payload) - - def test_missing_operation_id_in_operation(self, test_files_path): - """ - Test that the tool definition is generated correctly when the operationId is missing in the specification. - """ - config = ClientConfiguration(openapi_spec=create_openapi_spec(test_files_path / "yaml" / "openapi_edge_cases.yml"), - request_sender=FastAPITestClient(None)) - - tools = config.get_tools_definitions(), - tool_def = tools[0][0] - assert tool_def["type"] == "function" - assert tool_def["function"]["name"] == "missing_operation_id_get" - - def test_servers_order(self, test_files_path): - """ - Test that servers defined in different locations in the specification are used correctly. - """ - - config = ClientConfiguration(openapi_spec=create_openapi_spec(test_files_path / "yaml" / "openapi_edge_cases.yml"), - request_sender=FastAPITestClient(None)) - - op = config.openapi_spec.find_operation_by_id("servers_order_path") - assert op.get_server() == "https://inpath.example.com" - op = config.openapi_spec.find_operation_by_id("servers_order_operation") - assert op.get_server() == "https://inoperation.example.com" - op = config.openapi_spec.find_operation_by_id("missing_operation_id_get") - assert op.get_server() == "http://localhost" diff --git a/test/components/tools/openapi/test_openapi_client_error_handling.py b/test/components/tools/openapi/test_openapi_client_error_handling.py deleted file mode 100644 index c399c68d..00000000 --- a/test/components/tools/openapi/test_openapi_client_error_handling.py +++ /dev/null @@ -1,43 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 - - -import json - -import pytest -from fastapi import FastAPI, HTTPException - -from haystack_experimental.components.tools.openapi._openapi import OpenAPIServiceClient, HttpClientError, \ - ClientConfiguration -from test.components.tools.openapi.conftest import FastAPITestClient, create_openapi_spec - - -def create_error_handling_app() -> FastAPI: - app = FastAPI() - - @app.get("/error/{status_code}") - def raise_http_error(status_code: int): - raise HTTPException(status_code=status_code, detail=f"HTTP {status_code} error") - - return app - - -class TestErrorHandling: - @pytest.mark.parametrize("status_code", [400, 401, 403, 404, 500]) - def test_http_error_handling(self, test_files_path, status_code): - config = ClientConfiguration(openapi_spec=create_openapi_spec(test_files_path / "yaml" / "openapi_error_handling.yml"), - request_sender=FastAPITestClient(create_error_handling_app())) - client = OpenAPIServiceClient(config) - json_error = {"status_code": status_code} - payload = { - "type": "function", - "function": { - "arguments": json.dumps(json_error), - "name": "raiseHttpError", - }, - } - with pytest.raises(HttpClientError) as exc_info: - client.invoke(payload) - - assert str(status_code) in str(exc_info.value) diff --git a/test/components/tools/openapi/test_openapi_client_live.py b/test/components/tools/openapi/test_openapi_client_live.py deleted file mode 100644 index 97daa24a..00000000 --- a/test/components/tools/openapi/test_openapi_client_live.py +++ /dev/null @@ -1,48 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 - -import json -import os - -import pytest -import yaml -from haystack_experimental.components.tools.openapi._openapi import OpenAPIServiceClient, ClientConfiguration -from test.components.tools.openapi.conftest import create_openapi_spec - - -class TestClientLive: - - @pytest.mark.skipif(not os.environ.get("SERPERDEV_API_KEY", ""), reason="SERPERDEV_API_KEY not set or empty") - @pytest.mark.integration - def test_serperdev(self, test_files_path): - config = ClientConfiguration(openapi_spec=create_openapi_spec(test_files_path / "yaml" / "serper.yml"), credentials=os.getenv("SERPERDEV_API_KEY")) - serper_api = OpenAPIServiceClient(config) - payload = { - "id": "call_NJr1NBz2Th7iUWJpRIJZoJIA", - "function": { - "arguments": '{"q": "Who was Nikola Tesla?"}', - "name": "serperdev_search", - }, - "type": "function", - } - response = serper_api.invoke(payload) - assert "invention" in str(response) - - @pytest.mark.integration - @pytest.mark.unstable("This test hits rate limit on Github API.") - def test_github(self, test_files_path): - config = ClientConfiguration(openapi_spec=create_openapi_spec(test_files_path / "yaml" / "github_compare.yml")) - api = OpenAPIServiceClient(config) - - params = {"owner": "deepset-ai", "repo": "haystack", "basehead": "main...add_default_adapter_filters"} - payload = { - "id": "call_NJr1NBz2Th7iUWJpRIJZoJIA", - "function": { - "arguments": json.dumps(params), - "name": "compare_branches", - }, - "type": "function", - } - response = api.invoke(payload) - assert "deepset" in str(response) diff --git a/test/components/tools/openapi/test_openapi_client_live_anthropic.py b/test/components/tools/openapi/test_openapi_client_live_anthropic.py deleted file mode 100644 index ffd1daef..00000000 --- a/test/components/tools/openapi/test_openapi_client_live_anthropic.py +++ /dev/null @@ -1,64 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 - -import os - -import anthropic -import pytest - -from haystack_experimental.components.tools.openapi._openapi import ClientConfiguration, OpenAPIServiceClient, \ - LLMProvider -from test.components.tools.openapi.conftest import create_openapi_spec - - -class TestClientLiveAnthropic: - - @pytest.mark.skipif(not os.environ.get("SERPERDEV_API_KEY", ""), reason="SERPERDEV_API_KEY not set or empty") - @pytest.mark.skipif(not os.environ.get("ANTHROPIC_API_KEY", ""), reason="ANTHROPIC_API_KEY not set or empty") - @pytest.mark.integration - def test_serperdev(self, test_files_path): - config = ClientConfiguration(openapi_spec=create_openapi_spec(test_files_path / "yaml" / "serper.yml"), - credentials=os.getenv("SERPERDEV_API_KEY"), - llm_provider=LLMProvider.ANTHROPIC) - client = anthropic.Anthropic(api_key=os.getenv("ANTHROPIC_API_KEY")) - response = client.messages.create( - model="claude-3-opus-20240229", - max_tokens=1024, - tools=config.get_tools_definitions(), - messages=[{"role": "user", "content": "Do a google search: Who was Nikola Tesla?"}], - ) - service_api = OpenAPIServiceClient(config) - service_response = service_api.invoke(response) - assert "inventions" in str(service_response) - - # make a few more requests to test the same tool - service_response = service_api.invoke(response) - assert "Serbian" in str(service_response) - - service_response = service_api.invoke(response) - assert "American" in str(service_response) - - @pytest.mark.skipif(not os.environ.get("ANTHROPIC_API_KEY", ""), reason="ANTHROPIC_API_KEY not set or empty") - @pytest.mark.integration - @pytest.mark.unstable("This test hits rate limit on Github API.") - def test_github(self, test_files_path): - config = ClientConfiguration(openapi_spec=create_openapi_spec(test_files_path / "yaml" / "github_compare.yml"), - llm_provider=LLMProvider.ANTHROPIC) - - client = anthropic.Anthropic(api_key=os.getenv("ANTHROPIC_API_KEY")) - response = client.messages.create( - model="claude-3-opus-20240229", - max_tokens=1024, - tools=config.get_tools_definitions(), - messages=[ - { - "role": "user", - "content": "Compare branches main and add_default_adapter_filters in repo" - " haystack and owner deepset-ai", - } - ], - ) - service_api = OpenAPIServiceClient(config) - service_response = service_api.invoke(response) - assert "deepset" in str(service_response) diff --git a/test/components/tools/openapi/test_openapi_client_live_cohere.py b/test/components/tools/openapi/test_openapi_client_live_cohere.py deleted file mode 100644 index e5944abf..00000000 --- a/test/components/tools/openapi/test_openapi_client_live_cohere.py +++ /dev/null @@ -1,70 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 -import os - -import cohere -import pytest - -from haystack_experimental.components.tools.openapi._openapi import ClientConfiguration, OpenAPIServiceClient, \ - LLMProvider -from test.components.tools.openapi.conftest import create_openapi_spec - -# Copied from Cohere's documentation -preamble = """ -## Task & Context -You help people answer their questions and other requests interactively. You will be asked a very wide array of - requests on all kinds of topics. You will be equipped with a wide range of search engines or similar tools to - help you, which you use to research your answer. You should focus on serving the user's needs as best you can, - which will be wide-ranging. - -## Style Guide -Unless the user asks for a different style of answer, you should answer in full sentences, using proper grammar and - spelling. -""" - - -class TestClientLiveCohere: - - @pytest.mark.skipif(not os.environ.get("SERPERDEV_API_KEY", ""), reason="SERPERDEV_API_KEY not set or empty") - @pytest.mark.skipif(not os.environ.get("COHERE_API_KEY", ""), reason="COHERE_API_KEY not set or empty") - @pytest.mark.integration - def test_serperdev(self, test_files_path): - config = ClientConfiguration(openapi_spec=create_openapi_spec(test_files_path / "yaml" / "serper.yml"), - credentials=os.getenv("SERPERDEV_API_KEY"), - llm_provider=LLMProvider.COHERE) - client = cohere.Client(api_key=os.getenv("COHERE_API_KEY")) - response = client.chat( - model="command-r", - preamble=preamble, - tools=config.get_tools_definitions(), - message="Do a google search: Who was Nikola Tesla?", - ) - service_api = OpenAPIServiceClient(config) - service_response = service_api.invoke(response) - assert "inventions" in str(service_response) - - # make a few more requests to test the same tool - service_response = service_api.invoke(response) - assert "Serbian" in str(service_response) - - service_response = service_api.invoke(response) - assert "American" in str(service_response) - - @pytest.mark.skipif(not os.environ.get("COHERE_API_KEY", ""), reason="COHERE_API_KEY not set or empty") - @pytest.mark.integration - @pytest.mark.unstable("This test hits rate limit on Github API.") - def test_github(self, test_files_path): - config = ClientConfiguration(openapi_spec=create_openapi_spec(test_files_path / "yaml" / "github_compare.yml"), - llm_provider=LLMProvider.COHERE) - - client = cohere.Client(api_key=os.getenv("COHERE_API_KEY")) - response = client.chat( - model="command-r", - preamble=preamble, - tools=config.get_tools_definitions(), - message="Compare branches main and add_default_adapter_filters in repo haystack and owner deepset-ai", - ) - service_api = OpenAPIServiceClient(config) - service_response = service_api.invoke(response) - assert "deepset" in str(service_response) diff --git a/test/components/tools/openapi/test_openapi_client_live_openai.py b/test/components/tools/openapi/test_openapi_client_live_openai.py deleted file mode 100644 index 7a2d37c2..00000000 --- a/test/components/tools/openapi/test_openapi_client_live_openai.py +++ /dev/null @@ -1,95 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 - -import os - -import pytest -from openai import OpenAI - -from haystack_experimental.components.tools.openapi._openapi import ClientConfiguration, OpenAPIServiceClient -from test.components.tools.openapi.conftest import create_openapi_spec - - -class TestClientLiveOpenAPI: - - @pytest.mark.skipif(not os.environ.get("SERPERDEV_API_KEY", ""), reason="SERPERDEV_API_KEY not set or empty") - @pytest.mark.skipif(not os.environ.get("OPENAI_API_KEY", ""), reason="OPENAI_API_KEY not set or empty") - @pytest.mark.integration - def test_serperdev(self, test_files_path): - - config = ClientConfiguration(openapi_spec=create_openapi_spec(test_files_path / "yaml" / "serper.yml"), - credentials=os.getenv("SERPERDEV_API_KEY")) - client = OpenAI(api_key=os.getenv("OPENAI_API_KEY")) - response = client.chat.completions.create( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Do a serperdev google search: Who was Nikola Tesla?"}], - tools=config.get_tools_definitions(), - ) - service_api = OpenAPIServiceClient(config) - service_response = service_api.invoke(response) - assert "inventions" in str(service_response) - - # make a few more requests to test the same tool - service_response = service_api.invoke(response) - assert "Serbian" in str(service_response) - - service_response = service_api.invoke(response) - assert "American" in str(service_response) - - @pytest.mark.skipif(not os.environ.get("OPENAI_API_KEY", ""), reason="OPENAI_API_KEY not set or empty") - @pytest.mark.integration - @pytest.mark.unstable("This test hits rate limit on Github API.") - def test_github(self, test_files_path): - config = ClientConfiguration(openapi_spec=create_openapi_spec(test_files_path / "yaml" / "github_compare.yml")) - client = OpenAI(api_key=os.getenv("OPENAI_API_KEY")) - response = client.chat.completions.create( - model="gpt-3.5-turbo", - messages=[ - { - "role": "user", - "content": "Compare branches main and add_default_adapter_filters in repo" - " haystack and owner deepset-ai", - } - ], - tools=config.get_tools_definitions(), - ) - service_api = OpenAPIServiceClient(config) - service_response = service_api.invoke(response) - assert "deepset" in str(service_response) - - @pytest.mark.skipif(not os.environ.get("FIRECRAWL_API_KEY", ""), reason="FIRECRAWL_API_KEY not set or empty") - @pytest.mark.skipif(not os.environ.get("OPENAI_API_KEY", ""), reason="OPENAI_API_KEY not set or empty") - @pytest.mark.integration - @pytest.mark.unstable("This test is flaky likely due to load on the popular Firecrawl API") - def test_firecrawl(self): - openapi_spec_url = "https://raw.githubusercontent.com/mendableai/firecrawl/main/apps/api/openapi.json" - config = ClientConfiguration(openapi_spec=create_openapi_spec(openapi_spec_url), credentials=os.getenv("FIRECRAWL_API_KEY")) - client = OpenAI(api_key=os.getenv("OPENAI_API_KEY")) - response = client.chat.completions.create( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Scrape URL: https://news.ycombinator.com/"}], - tools=config.get_tools_definitions(), - ) - service_api = OpenAPIServiceClient(config) - service_response = service_api.invoke(response) - assert isinstance(service_response, dict) - assert service_response.get("success", False), "Firecrawl scrape API call failed" - - # now test the same openapi service but different endpoint/tool - top_k = 2 - response = client.chat.completions.create( - model="gpt-3.5-turbo", - messages=[ - { - "role": "user", - "content": f"Search Google for `Why was Sam Altman ousted from OpenAI?`, limit to {top_k} results", - } - ], - tools=config.get_tools_definitions(), - ) - service_response = service_api.invoke(response) - assert isinstance(service_response, dict) - assert service_response.get("success", False), "Firecrawl search API call failed" - assert len(service_response.get("data", [])) == top_k - assert "Sam" in str(service_response) diff --git a/test/components/tools/openapi/test_openapi_cohere_conversion.py b/test/components/tools/openapi/test_openapi_cohere_conversion.py deleted file mode 100644 index ac8f42b2..00000000 --- a/test/components/tools/openapi/test_openapi_cohere_conversion.py +++ /dev/null @@ -1,164 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 - -from haystack_experimental.components.tools.openapi._openapi import OpenAPISpecification, cohere_converter - - -class TestOpenAPISchemaConversion: - - def test_serperdev(self, test_files_path): - spec = OpenAPISpecification.from_file(test_files_path / "yaml" / "serper.yml") - functions = cohere_converter(schema=spec) - - assert functions - assert len(functions) == 1 - function = functions[0] - assert function["name"] == "serperdev_search" - assert function["description"] == "Search the web with Google" - assert function["parameter_definitions"] == { - "q": {"description": "", "type": "str", "required": True} - } - - def test_firecrawler(self, test_files_path): - spec = OpenAPISpecification.from_file( - test_files_path / "json" / "firecrawl_openapi_spec.json" - ) - functions = cohere_converter(schema=spec) - assert functions - assert len(functions) == 5 - function = functions[0] - assert function["name"] == "scrapeAndExtractFromUrl" - assert ( - function["description"] - == "Scrape a single URL and optionally extract information using an LLM" - ) - assert function["parameter_definitions"] == { - "url": {"type": "str", "description": "The URL to scrape", "required": True}, - "pageOptions": { - "type": "object", - "description": "", - "required": False, - "properties": { - "onlyMainContent": { - "type": "bool", - "description": "Only return the main content of the page excluding headers, navs, footers, etc.", - "required": False, - }, - "includeHtml": { - "type": "bool", - "description": "Include the raw HTML content of the page. Will output a html key in the response.", - "required": False, - }, - "screenshot": { - "type": "bool", - "description": "Include a screenshot of the top of the page that you are scraping.", - "required": False, - }, - "waitFor": { - "type": "int", - "description": "Wait x amount of milliseconds for the page to load to fetch content", - "required": False, - }, - "removeTags": { - "type": "list", - "description": "Tags, classes and ids to remove from the page. Use comma separated values. Example: 'script, .ad, #footer'", - "required": False, - }, - "headers": { - "type": "object", - "description": "Headers to send with the request. Can be used to send cookies, user-agent, etc.", - "properties": {}, - "required": False, - }, - }, - }, - "extractorOptions": { - "type": "object", - "description": "Options for LLM-based extraction of structured information from the page content", - "required": False, - "properties": { - "mode": { - "type": "str", - "description": "The extraction mode to use, currently supports 'llm-extraction'", - "required": False, - }, - "extractionPrompt": { - "type": "str", - "description": "A prompt describing what information to extract from the page", - "required": False, - }, - "extractionSchema": { - "type": "object", - "description": "The schema for the data to be extracted", - "properties": {}, - "required": False, - }, - }, - }, - "timeout": { - "type": "int", - "description": "Timeout in milliseconds for the request", - "required": False, - }, - } - - def test_github(self, test_files_path): - spec = OpenAPISpecification.from_file(test_files_path / "yaml" / "github_compare.yml") - functions = cohere_converter(schema=spec) - assert functions - assert len(functions) == 1 - function = functions[0] - assert function["name"] == "compare_branches" - assert function["description"] == "Compares two branches against one another." - assert function["parameter_definitions"] == { - "basehead": { - "description": "The base branch and head branch to compare." - " This parameter expects the format `BASE...HEAD`", - "type": "str", - "required": True, - }, - "owner": { - "description": "The repository owner, usually a company or orgnization", - "type": "str", - "required": True, - }, - "repo": {"description": "The repository itself, the project", "type": "str", "required": True}, - } - - def test_complex_types(self, test_files_path): - spec = OpenAPISpecification.from_file(test_files_path / "json" / "complex_types_openapi_service.json") - functions = cohere_converter(schema=spec) - - assert functions - assert len(functions) == 1 - function = functions[0] - assert function["name"] == "processPayment" - assert function["description"] == "Process a new payment using the specified payment method" - assert function["parameter_definitions"] == { - "transaction_amount": {"type": "float", "description": "The amount to be paid", "required": True}, - "description": {"type": "str", "description": "A brief description of the payment", "required": True}, - "payment_method_id": {"type": "str", "description": "The payment method to be used", "required": True}, - "payer": { - "type": "object", - "description": "Information about the payer, including their name, email, and identification number", - "properties": { - "name": {"type": "str", "description": "The payer's name", "required": True}, - "email": {"type": "str", "description": "The payer's email address", "required": True}, - "identification": { - "type": "object", - "description": "The payer's identification number", - "properties": { - "type": { - "type": "str", - "description": "The type of identification document (e.g., CPF, CNPJ)", - "required": True, - }, - "number": {"type": "str", "description": "The identification number", "required": True}, - }, - "required": True, - }, - }, - "required": True, - }, - } diff --git a/test/components/tools/openapi/test_openapi_openai_conversion.py b/test/components/tools/openapi/test_openapi_openai_conversion.py deleted file mode 100644 index 9e7285dc..00000000 --- a/test/components/tools/openapi/test_openapi_openai_conversion.py +++ /dev/null @@ -1,108 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 - -import pytest - -from haystack_experimental.components.tools.openapi._openapi import ( - openai_converter, - anthropic_converter, - OpenAPISpecification, -) - - -class TestOpenAPISchemaConversion: - - @pytest.mark.parametrize("provider", ["openai", "anthropic"]) - def test_serperdev(self, test_files_path, provider): - spec = OpenAPISpecification.from_file(test_files_path / "yaml" / "serper.yml") - functions = openai_converter(schema=spec) if provider == "openai" else anthropic_converter(schema=spec) - assert functions - assert len(functions) == 1 - function = functions[0]["function"] if provider == "openai" else functions[0] - assert function["name"] == "serperdev_search" - assert function["description"] == "Search the web with Google" - assert ( - function["parameters"] - if provider == "openai" - else function["input_schema"] - == {"type": "object", "properties": {"q": {"type": "string"}}, "required": ["q"]} - ) - - @pytest.mark.parametrize("provider", ["openai", "anthropic"]) - def test_github(self, test_files_path, provider: str): - spec = OpenAPISpecification.from_file(test_files_path / "yaml" / "github_compare.yml") - functions = openai_converter(schema=spec) if provider == "openai" else anthropic_converter(schema=spec) - assert functions - assert len(functions) == 1 - function = functions[0]["function"] if provider == "openai" else functions[0] - assert function["name"] == "compare_branches" - assert function["description"] == "Compares two branches against one another." - assert ( - function["parameters"] - if provider == "openai" - else function["input_schema"] - == { - "type": "object", - "properties": { - "basehead": { - "type": "string", - "description": "The base branch and head branch to compare. " - "This parameter expects the format `BASE...HEAD`", - }, - "owner": { - "type": "string", - "description": "The repository owner, usually a company or orgnization", - }, - "repo": {"type": "string", "description": "The repository itself, the project"}, - }, - "required": ["basehead", "owner", "repo"], - } - ) - - @pytest.mark.parametrize("provider", ["openai", "anthropic"]) - def test_complex_types(self, test_files_path, provider: str): - spec = OpenAPISpecification.from_file(test_files_path / "json" / "complex_types_openapi_service.json") - functions = openai_converter(schema=spec) if provider == "openai" else anthropic_converter(schema=spec) - - assert functions - assert len(functions) == 1 - function = functions[0]["function"] if provider == "openai" else functions[0] - assert function["name"] == "processPayment" - assert function["description"] == "Process a new payment using the specified payment method" - assert ( - function["parameters"] - if provider == "openai" - else function["input_schema"] - == { - "type": "object", - "properties": { - "transaction_amount": {"type": "number", "description": "The amount to be paid"}, - "description": {"type": "string", "description": "A brief description of the payment"}, - "payment_method_id": {"type": "string", "description": "The payment method to be used"}, - "payer": { - "type": "object", - "description": "Information about the payer, including their name, email, " - "and identification number", - "properties": { - "name": {"type": "string", "description": "The payer's name"}, - "email": {"type": "string", "description": "The payer's email address"}, - "identification": { - "type": "object", - "description": "The payer's identification number", - "properties": { - "type": { - "type": "string", - "description": "The type of identification document (e.g., CPF, CNPJ)", - }, - "number": {"type": "string", "description": "The identification number"}, - }, - "required": ["type", "number"], - }, - }, - "required": ["name", "email", "identification"], - }, - }, - "required": ["transaction_amount", "description", "payment_method_id", "payer"], - } - ) diff --git a/test/components/tools/openapi/test_openapi_spec.py b/test/components/tools/openapi/test_openapi_spec.py deleted file mode 100644 index 4e38de2d..00000000 --- a/test/components/tools/openapi/test_openapi_spec.py +++ /dev/null @@ -1,76 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 - -import pytest - -from haystack_experimental.components.tools.openapi._openapi import OpenAPISpecification - - -class TestOpenAPISpecification: - - # can be initialized from a string - def test_initialized_from_string(self): - content = """ - openapi: 3.0.0 - info: - title: Test API - version: 1.0.0 - servers: - - url: https://api.example.com - paths: - /users: - get: - summary: Get all users - responses: - '200': - description: Successful response - """ - openapi_spec = OpenAPISpecification.from_str(content) - assert openapi_spec.spec_dict == { - "openapi": "3.0.0", - "info": {"title": "Test API", "version": "1.0.0"}, - "servers": [{"url": "https://api.example.com"}], - "paths": { - "/users": { - "get": {"summary": "Get all users", "responses": {"200": {"description": "Successful response"}}} - } - }, - } - - # can be initialized from a file - def test_initialized_from_file(self, tmp_path): - content = """ - openapi: 3.0.0 - info: - title: Test API - version: 1.0.0 - servers: - - url: https://api.example.com - paths: - /users: - get: - summary: Get all users - responses: - '200': - description: Successful response - """ - file_path = tmp_path / "spec.yaml" - file_path.write_text(content) - openapi_spec = OpenAPISpecification.from_file(file_path) - assert openapi_spec.spec_dict == { - "openapi": "3.0.0", - "info": {"title": "Test API", "version": "1.0.0"}, - "servers": [{"url": "https://api.example.com"}], - "paths": { - "/users": { - "get": {"summary": "Get all users", "responses": {"200": {"description": "Successful response"}}} - } - }, - } - - # raises ValueError if initialized from an invalid schema - def test_raises_value_error_invalid_schema(self): - spec_dict = {"info": {"title": "Test API", "version": "1.0.0"}, "paths": {"/users": {}}} - with pytest.raises(ValueError): - OpenAPISpecification(spec_dict) diff --git a/test/components/tools/openapi/test_openapi_tool.py b/test/components/tools/openapi/test_openapi_tool.py deleted file mode 100644 index bf3ca57e..00000000 --- a/test/components/tools/openapi/test_openapi_tool.py +++ /dev/null @@ -1,293 +0,0 @@ -import json -import os - -from haystack.components.generators.chat import OpenAIChatGenerator -from haystack.dataclasses import ChatMessage -from haystack.utils import Secret - -from haystack_experimental.components.tools.openapi import LLMProvider -from haystack_experimental.components.tools.openapi.openapi_tool import OpenAPITool - -import pytest - -from .conftest import provider_api_key_set - -class TestOpenAPITool: - - def test_to_dict(self, monkeypatch): - monkeypatch.setenv("OPENAI_API_KEY", "test-api-key") - monkeypatch.setenv("SERPERDEV_API_KEY", "fake-api-key") - - openapi_spec_url = "https://raw.githubusercontent.com/mendableai/firecrawl/main/apps/api/openapi.json" - - tool = OpenAPITool( - generator_api=LLMProvider.OPENAI, - generator_api_params={ - "model": "gpt-4o-mini", - "api_key": Secret.from_env_var("OPENAI_API_KEY"), - }, - spec=openapi_spec_url, - credentials=Secret.from_env_var("SERPERDEV_API_KEY"), - allowed_operations=["someOperationId", "someOtherOperationId"], - ) - - data = tool.to_dict() - assert data == { - "type": "haystack_experimental.components.tools.openapi.openapi_tool.OpenAPITool", - "init_parameters": { - "generator_api": "openai", - "generator_api_params": { - "model": "gpt-4o-mini", - "api_key": {"env_vars": ["OPENAI_API_KEY"], "strict": True, "type": "env_var"}, - }, - "spec": openapi_spec_url, - "credentials": {"env_vars": ["SERPERDEV_API_KEY"], "strict": True, "type": "env_var"}, - "allowed_operations": ["someOperationId", "someOtherOperationId"], - }, - } - - def test_from_dict(self, monkeypatch): - monkeypatch.setenv("OPENAI_API_KEY", "fake-api-key") - monkeypatch.setenv("SERPERDEV_API_KEY", "fake-api-key") - openapi_spec_url = "https://raw.githubusercontent.com/mendableai/firecrawl/main/apps/api/openapi.json" - data = { - "type": "haystack_experimental.components.tools.openapi.openapi_tool.OpenAPITool", - "init_parameters": { - "generator_api": "openai", - "generator_api_params": { - "model": "gpt-4o-mini", - "api_key": {"env_vars": ["OPENAI_API_KEY"], "strict": True, "type": "env_var"}, - }, - "spec": openapi_spec_url, - "credentials": {"env_vars": ["SERPERDEV_API_KEY"], "strict": True, "type": "env_var"}, - "allowed_operations": None, - }, - } - - tool = OpenAPITool.from_dict(data) - - assert tool.generator_api == LLMProvider.OPENAI - assert tool.generator_api_params == { - "model": "gpt-4o-mini", - "api_key": Secret.from_env_var("OPENAI_API_KEY") - } - assert tool.spec == openapi_spec_url - assert tool.credentials == Secret.from_env_var("SERPERDEV_API_KEY") - - def test_initialize_with_invalid_openapi_spec_url(self): - with pytest.raises(ConnectionError, match="Failed to fetch the specification from URL"): - OpenAPITool( - generator_api=LLMProvider.OPENAI, - generator_api_params={ - "model": "gpt-4o-mini", - "api_key": Secret.from_token("not_needed"), - }, - spec="https://raw.githubusercontent.com/invalid_openapi.json", - ) - - def test_initialize_with_invalid_openapi_spec_path(self): - with pytest.raises(ValueError, match="Invalid OpenAPI specification source"): - OpenAPITool( - generator_api=LLMProvider.OPENAI, - generator_api_params={ - "model": "gpt-4o-mini", - "api_key": Secret.from_token("not_needed"), - }, - spec="invalid_openapi.json", - ) - - def test_initialize_with_valid_openapi_spec_url_and_credentials(self): - openapi_spec_url = "https://raw.githubusercontent.com/mendableai/firecrawl/main/apps/api/openapi.json" - credentials = Secret.from_token("") - tool = OpenAPITool( - generator_api=LLMProvider.OPENAI, - generator_api_params={ - "model": "gpt-4o-mini", - "api_key": Secret.from_token("not_needed"), - }, - spec=openapi_spec_url, - credentials=credentials, - ) - - assert tool.generator_api == LLMProvider.OPENAI - assert isinstance(tool.chat_generator, OpenAIChatGenerator) - assert tool.config_openapi is not None - assert tool.open_api_service is not None - - @pytest.mark.skipif(not os.environ.get("SERPERDEV_API_KEY", ""), reason="SERPERDEV_API_KEY not set or empty") - @pytest.mark.skipif(not os.environ.get("OPENAI_API_KEY", ""), reason="OPENAI_API_KEY not set or empty") - @pytest.mark.integration - def test_run_live_openai(self): - tool = OpenAPITool( - generator_api=LLMProvider.OPENAI, - spec="https://bit.ly/serper_dev_spec_yaml", - credentials=Secret.from_env_var("SERPERDEV_API_KEY"), - ) - - user_message = ChatMessage.from_user( - "Search for 'Who was Nikola Tesla?'" - ) - - results = tool.run(messages=[user_message]) - - assert isinstance(results["service_response"], list) - assert len(results["service_response"]) == 1 - assert isinstance(results["service_response"][0], ChatMessage) - - try: - json_response = json.loads(results["service_response"][0].content) - assert isinstance(json_response, dict) - except json.JSONDecodeError: - pytest.fail("Response content is not valid JSON") - - @pytest.mark.skipif(not os.environ.get("SERPERDEV_API_KEY", ""), reason="SERPERDEV_API_KEY not set or empty") - @pytest.mark.skipif(not os.environ.get("ANTHROPIC_API_KEY", ""), reason="ANTHROPIC_API_KEY not set or empty") - @pytest.mark.integration - def test_run_live_anthropic(self): - tool = OpenAPITool( - generator_api=LLMProvider.ANTHROPIC, - generator_api_params={"model": "claude-3-opus-20240229"}, - spec="https://bit.ly/serper_dev_spec_yaml", - credentials=Secret.from_env_var("SERPERDEV_API_KEY"), - ) - - user_message = ChatMessage.from_user( - "Search for 'Who was Nikola Tesla?'" - ) - - results = tool.run(messages=[user_message]) - - assert isinstance(results["service_response"], list) - assert len(results["service_response"]) == 1 - assert isinstance(results["service_response"][0], ChatMessage) - - try: - json_response = json.loads(results["service_response"][0].content) - assert isinstance(json_response, dict) - except json.JSONDecodeError: - pytest.fail("Response content is not valid JSON") - - @pytest.mark.skipif(not os.environ.get("SERPERDEV_API_KEY", ""), reason="SERPERDEV_API_KEY not set or empty") - @pytest.mark.skipif(not os.environ.get("COHERE_API_KEY", ""), reason="COHERE_API_KEY not set or empty") - @pytest.mark.integration - def test_run_live_cohere(self): - tool = OpenAPITool( - generator_api=LLMProvider.COHERE, - generator_api_params={"model": "command-r"}, - spec="https://bit.ly/serper_dev_spec_yaml", - credentials=Secret.from_env_var("SERPERDEV_API_KEY"), - ) - - user_message = ChatMessage.from_user( - "Search for 'Who was Nikola Tesla?'" - ) - - results = tool.run(messages=[user_message]) - - assert isinstance(results["service_response"], list) - assert len(results["service_response"]) == 1 - assert isinstance(results["service_response"][0], ChatMessage) - - try: - json_response = json.loads(results["service_response"][0].content) - assert isinstance(json_response, dict) - except json.JSONDecodeError: - pytest.fail("Response content is not valid JSON") - - @pytest.mark.integration - @pytest.mark.parametrize("provider", ["openai", "anthropic", "cohere"]) - @pytest.mark.unstable("This test can be unstable due to free meteo service being down") - def test_run_live_meteo_forecast(self, provider: str): - if not provider_api_key_set(provider): - pytest.skip(f"API key for {provider} is not set") - - tool = OpenAPITool( - generator_api=LLMProvider.from_str(provider), - spec="https://raw.githubusercontent.com/open-meteo/open-meteo/main/openapi.yml" - ) - results = tool.run(messages=[ChatMessage.from_user( - "weather forecast for latitude 52.52 and longitude 13.41 and set hourly=temperature_2m")]) - - assert isinstance(results["service_response"], list) - assert len(results["service_response"]) == 1 - assert isinstance(results["service_response"][0], ChatMessage) - - try: - json_response = json.loads(results["service_response"][0].content) - assert isinstance(json_response, dict) - assert "hourly" in json_response - except json.JSONDecodeError: - pytest.fail("Response content is not valid JSON") - - @pytest.mark.integration - @pytest.mark.parametrize("provider", ["openai", "anthropic", "cohere"]) - @pytest.mark.unstable("This test can be unstable due to free meteo service being down") - def test_run_live_meteo_forecast_with_non_normalized_operation_id(self, provider: str): - """ - Test that OpenAPITool can handle non-normalized operationIds (function names not accepted by LLMs). - Here we test all the supported LLMs. - """ - if not provider_api_key_set(provider): - pytest.skip(f"API key for {provider} is not set") - tool = OpenAPITool( - generator_api=LLMProvider.from_str(provider), - spec="https://bit.ly/meteo_with_non_normalized_operationId" - ) - results = tool.run(messages=[ChatMessage.from_user( - "weather forecast for latitude 52.52 and longitude 13.41 and set hourly=temperature_2m")]) - - assert isinstance(results["service_response"], list) - assert len(results["service_response"]) == 1 - assert isinstance(results["service_response"][0], ChatMessage) - - try: - json_response = json.loads(results["service_response"][0].content) - assert isinstance(json_response, dict) - assert "hourly" in json_response - except json.JSONDecodeError: - pytest.fail("Response content is not valid JSON") - - @pytest.mark.skipif(not os.environ.get("OPENAI_API_KEY", ""), reason="OPENAI_API_KEY not set or empty") - def test_allowed_operations(self): - """ - Although the tool definition is generated from the OpenAPI spec and firecrawl's API has multiple operations, - only the ones we specify in the allowed_operations list are registered with LLMs via the tool definition. - """ - tool = OpenAPITool( - generator_api=LLMProvider.OPENAI, - spec="https://raw.githubusercontent.com/mendableai/firecrawl/main/apps/api/openapi.json", - allowed_operations=["scrape"], - ) - tools = tool.config_openapi.get_tools_definitions() - assert len(tools) == 1 - assert tools[0]["function"]["name"] == "scrape" - - # test two operations - tool = OpenAPITool( - generator_api=LLMProvider.OPENAI, - spec="https://raw.githubusercontent.com/mendableai/firecrawl/main/apps/api/openapi.json", - allowed_operations=["scrape", "crawlUrls"], - ) - tools = tool.config_openapi.get_tools_definitions() - assert len(tools) == 2 - assert tools[0]["function"]["name"] == "scrape" - assert tools[1]["function"]["name"] == "crawlUrls" - - # test non-existent operation - tool = OpenAPITool( - generator_api=LLMProvider.OPENAI, - spec="https://raw.githubusercontent.com/mendableai/firecrawl/main/apps/api/openapi.json", - allowed_operations=["scrape", "non-existent-operation"], - ) - tools = tool.config_openapi.get_tools_definitions() - assert len(tools) == 1 - assert tools[0]["function"]["name"] == "scrape" - - # test all non-existent operations - tool = OpenAPITool( - generator_api=LLMProvider.OPENAI, - spec="https://raw.githubusercontent.com/mendableai/firecrawl/main/apps/api/openapi.json", - allowed_operations=["non-existent-operation", "non-existent-operation-2"], - ) - tools = tool.config_openapi.get_tools_definitions() - assert len(tools) == 0 diff --git a/test/components/tools/openapi/test_util.py b/test/components/tools/openapi/test_util.py deleted file mode 100644 index 901d02f4..00000000 --- a/test/components/tools/openapi/test_util.py +++ /dev/null @@ -1,36 +0,0 @@ -from haystack_experimental.components.tools.utils import normalize_tool_definition, normalize_function_name - - -def test_normalize_function_name(): - assert normalize_function_name("test-function") == "test_function" - assert normalize_function_name("missing-operation-id_get") == "missing_operation_id_get" - assert normalize_function_name("/test/function/with/slashes-and-dashes") == "test_function_with_slashes_and_dashes" - assert normalize_function_name("test\\function\\with\\backslashes") == "test_function_with_backslashes" - assert normalize_function_name("-test-function-with-dashes-") == "test_function_with_dashes" - - -def test_normalize_tool_definition(): - function = { - "name": "_test-function/with/slashes-and-dashes_", - "description": "Test function description at least 50 characters long " * 100, - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA" * 100, - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location", "format"], - } - } - sanitized_function = normalize_tool_definition(function) - assert sanitized_function["name"] == "test_function_with_slashes_and_dashes" - assert len(sanitized_function["description"]) <= 1024 - assert len(sanitized_function["parameters"]["properties"]["location"]["description"]) <= 1024 - assert len(sanitized_function["parameters"]["properties"]["format"]["description"]) <= 1024