chore: Move OpenAPI test dependencies to hatch test (#23)

* Move OpenAPI test dependencies to hatch test

* Move jsonref into lazyimport guard

* Use import checks, unrelated small pydoc update

* Update deps to use haystack core integrations

* Add env vars to tests

* More env vars for integration tests

* Disable github tests - hitting quotas there

.github/workflows/tests.yml

@@ -28,6 +28,13 @@ on:
 env:
   PYTHON_VERSION: "3.8"
   HATCH_VERSION: "1.9.4"
+  PYTHONUNBUFFERED: "1"
+  FORCE_COLOR: "1"
+  ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+  COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
+  FIRECRAWL_API_KEY: ${{ secrets.FIRECRAWL_API_KEY }}
+  SERPERDEV_API_KEY: ${{ secrets.SERPERDEV_API_KEY }}
 
 jobs:
   linting:

README.md

@@ -36,11 +36,11 @@ that includes it. Once it reaches the end of its lifespan, the experiment will b
 
 The latest version of the package contains the following experiments:
 
-| Name                     | Type                    | Expected experiment end date |
-| ------------------------ | ----------------------- | ------------------- |
-| [`EvaluationHarness`][1] | Evaluation orchestrator | September 2024         |
-| [`OpenAIFunctionCaller`][2] | Function Calling Component | September 2024         |
-| [`OpenAPITool`][3]       | OpenAPITool component   | September 2024         |
+| Name                     | Type                    | Expected experiment end date | Dependencies |
+| ------------------------ | ----------------------- | ------------------- | ------------------- |
+| [`EvaluationHarness`][1] | Evaluation orchestrator | September 2024         | None
+| [`OpenAIFunctionCaller`][2] | Function Calling Component | September 2024         | None
+| [`OpenAPITool`][3]       | OpenAPITool component   | September 2024         | jsonref
 
 [1]: https://github.com/deepset-ai/haystack-experimental/tree/main/haystack_experimental/evaluation/harness
 [2]: https://github.com/deepset-ai/haystack-experimental/tree/main/haystack_experimental/components/tools/openai

haystack_experimental/components/tools/openapi/_schema_conversion.py

@@ -5,10 +5,15 @@
 import logging
 from typing import Any, Callable, Dict, List, Optional
 
-import jsonref
+from haystack.lazy_imports import LazyImport
 
 from haystack_experimental.components.tools.openapi.types import OpenAPISpecification
 
+with LazyImport("Run 'pip install jsonref'") as jsonref_import:
+    # pylint: disable=import-error
+    import jsonref
+
+
 MIN_REQUIRED_OPENAPI_SPEC_VERSION = 3
 
 logger = logging.getLogger(__name__)
@@ -22,6 +27,7 @@ def openai_converter(schema: OpenAPISpecification) -> List[Dict[str, Any]]:
     :param schema: The OpenAPI specification to convert.
     :returns: A list of dictionaries, each dictionary representing an OpenAI function definition.
     """
+    jsonref_import.check()
     resolved_schema = jsonref.replace_refs(schema.spec_dict)
     fn_definitions = _openapi_to_functions(
         resolved_schema, "parameters", _parse_endpoint_spec_openai
@@ -38,6 +44,7 @@ def anthropic_converter(schema: OpenAPISpecification) -> List[Dict[str, Any]]:
     :param schema: The OpenAPI specification to convert.
     :returns: A list of dictionaries, each dictionary representing Anthropic function definition.
     """
+    jsonref_import.check()
     resolved_schema = jsonref.replace_refs(schema.spec_dict)
     return _openapi_to_functions(
         resolved_schema, "input_schema", _parse_endpoint_spec_openai
@@ -53,6 +60,7 @@ def cohere_converter(schema: OpenAPISpecification) -> List[Dict[str, Any]]:
     :param schema: The OpenAPI specification to convert.
     :returns: A list of dictionaries, each representing a Cohere style function definition.
     """
+    jsonref_import.check()
     resolved_schema = jsonref.replace_refs(schema.spec_dict)
     return _openapi_to_functions(
         resolved_schema, "not important for cohere", _parse_endpoint_spec_cohere

haystack_experimental/components/tools/openapi/openapi_tool.py

@@ -47,7 +47,7 @@ class OpenAPITool:
     tool = OpenAPITool(generator_api=LLMProvider.OPENAI,
                        generator_api_params={"model":"gpt-3.5-turbo"},
                        spec="https://raw.githubusercontent.com/mendableai/firecrawl/main/apps/api/openapi.json",
-                       credentials=Secret.from_token("<your-tool-token>"))
+                       credentials=Secret.from_env_var("FIRECRAWL_API_KEY"))
 
     results = tool.run(messages=[ChatMessage.from_user("Scrape URL: https://news.ycombinator.com/")])
     print(results)

pyproject.toml

@@ -25,7 +25,7 @@ classifiers = [
   "Programming Language :: Python :: 3.12",
   "Topic :: Scientific/Engineering :: Artificial Intelligence",
 ]
-dependencies = ["jsonref", "haystack-ai"]
+dependencies = ["haystack-ai"]
 [project.urls]
 "CI: GitHub" = "https://github.com/deepset-ai/haystack-experimental/actions"
 "GitHub: issues" = "https://github.com/deepset-ai/haystack-experimental/issues"
@@ -39,9 +39,6 @@ dependencies = [
   # Test
   "pytest",
   "pytest-cov",
-  "fastapi",
-  "cohere",
-  "anthropic",
   # Linting
   "pylint",
   "ruff",
@@ -51,6 +48,12 @@ dependencies = [
 extra-dependencies = [
   # RAG evaluation harness (SAS evaluator)
   "sentence-transformers>=2.2.0",
+  # OpenAPI dependencies
+  "jsonref",
+  # OpenAPI tests
+  "cohere-haystack",
+  "anthropic-haystack",
+  "fastapi",
 ]
 
 [tool.hatch.envs.test.scripts]

test/components/tools/openapi/test_openapi_client_live_anthropic.py

@@ -41,6 +41,7 @@ def test_serperdev(self, test_files_path):
 
     @pytest.mark.skipif("ANTHROPIC_API_KEY" not in os.environ, reason="ANTHROPIC_API_KEY not set")
     @pytest.mark.integration
+    @pytest.mark.skip("This test hits rate limit on Github API. Skip for now.")
     def test_github(self, test_files_path):
         config = ClientConfiguration(openapi_spec=create_openapi_spec(test_files_path / "yaml" / "github_compare.yml"),
                                      llm_provider=LLMProvider.ANTHROPIC)

test/components/tools/openapi/test_openapi_client_live_cohere.py

@@ -53,6 +53,7 @@ def test_serperdev(self, test_files_path):
 
     @pytest.mark.skipif("COHERE_API_KEY" not in os.environ, reason="COHERE_API_KEY not set")
     @pytest.mark.integration
+    @pytest.mark.skip("This test hits rate limit on Github API. Skip for now.")
     def test_github(self, test_files_path):
         config = ClientConfiguration(openapi_spec=create_openapi_spec(test_files_path / "yaml" / "github_compare.yml"),
                                      llm_provider=LLMProvider.COHERE)