From 9014494d40a17f0678ccc45f8d51ffac1be514cf Mon Sep 17 00:00:00 2001 From: Vladimir Blagojevic Date: Tue, 30 Jan 2024 17:33:41 +0100 Subject: [PATCH] chore: Amazon Bedrock subproject refactoring (#293) * Initial migration * Update README.md * Update test instructions * Update integrations/amazon_bedrock/pyproject.toml Co-authored-by: Silvano Cerza <3314350+silvanocerza@users.noreply.github.com> * Update integrations/amazon_bedrock/pyproject.toml Co-authored-by: Silvano Cerza <3314350+silvanocerza@users.noreply.github.com> * Linting --------- Co-authored-by: Silvano Cerza <3314350+silvanocerza@users.noreply.github.com> --- integrations/amazon_bedrock/README.md | 19 ++++++++++++++++ integrations/amazon_bedrock/pyproject.toml | 21 ++++++++++-------- .../generators/__init__.py | 3 --- .../generators/amazon_bedrock}/__init__.py | 2 +- .../amazon_bedrock_adapters.py | 2 +- .../amazon_bedrock_handlers.py | 0 .../generators/amazon_bedrock}/errors.py | 0 .../generators/amazon_bedrock/generator.py} | 14 ++++++------ .../tests/test_amazon_bedrock.py | 22 +++++++++---------- 9 files changed, 51 insertions(+), 32 deletions(-) delete mode 100644 integrations/amazon_bedrock/src/amazon_bedrock_haystack/generators/__init__.py rename integrations/amazon_bedrock/src/{amazon_bedrock_haystack => haystack_integrations/components/generators/amazon_bedrock}/__init__.py (63%) rename integrations/amazon_bedrock/src/{amazon_bedrock_haystack/generators => haystack_integrations/components/generators/amazon_bedrock}/amazon_bedrock_adapters.py (98%) rename integrations/amazon_bedrock/src/{amazon_bedrock_haystack/generators => haystack_integrations/components/generators/amazon_bedrock}/amazon_bedrock_handlers.py (100%) rename integrations/amazon_bedrock/src/{amazon_bedrock_haystack => haystack_integrations/components/generators/amazon_bedrock}/errors.py (100%) rename integrations/amazon_bedrock/src/{amazon_bedrock_haystack/generators/amazon_bedrock.py => haystack_integrations/components/generators/amazon_bedrock/generator.py} (98%) diff --git a/integrations/amazon_bedrock/README.md b/integrations/amazon_bedrock/README.md index f84c8f3c4..3a689ef3b 100644 --- a/integrations/amazon_bedrock/README.md +++ b/integrations/amazon_bedrock/README.md @@ -8,6 +8,7 @@ **Table of Contents** - [Installation](#installation) +- [Contributing](#contributing) - [License](#license) ## Installation @@ -16,6 +17,24 @@ pip install amazon-bedrock-haystack ``` +## Contributing + +`hatch` is the best way to interact with this project, to install it: +```sh +pip install hatch +``` + +With `hatch` installed, to run all the tests: +``` +hatch run test +``` +> Note: there are no integration tests for this project. + +To run the linters `ruff` and `mypy`: +``` +hatch run lint:all +``` + ## License `amazon-bedrock-haystack` is distributed under the terms of the [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) license. diff --git a/integrations/amazon_bedrock/pyproject.toml b/integrations/amazon_bedrock/pyproject.toml index 7e82924a8..6a2ce3eab 100644 --- a/integrations/amazon_bedrock/pyproject.toml +++ b/integrations/amazon_bedrock/pyproject.toml @@ -35,6 +35,9 @@ Documentation = "https://github.com/deepset-ai/haystack-core-integrations/tree/m Issues = "https://github.com/deepset-ai/haystack-core-integrations/issues" Source = "https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/amazon_bedrock" +[tool.hatch.build.targets.wheel] +packages = ["src/haystack_integrations"] + [tool.hatch.version] source = "vcs" tag-pattern = 'integrations\/amazon_bedrock-v(?P.*)' @@ -71,7 +74,8 @@ dependencies = [ "ruff>=0.0.243", ] [tool.hatch.envs.lint.scripts] -typing = "mypy --install-types --non-interactive {args:src/amazon_bedrock_haystack tests}" +typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}" + style = [ "ruff {args:.}", "black --check --diff {args:.}", @@ -136,26 +140,24 @@ unfixable = [ ] [tool.ruff.isort] -known-first-party = ["amazon_bedrock_haystack"] +known-first-party = ["haystack_integrations"] [tool.ruff.flake8-tidy-imports] -ban-relative-imports = "all" +ban-relative-imports = "parents" [tool.ruff.per-file-ignores] # Tests can use magic values, assertions, and relative imports "tests/**/*" = ["PLR2004", "S101", "TID252"] [tool.coverage.run] -source_pkgs = ["amazon_bedrock_haystack", "tests"] +source_pkgs = ["src", "tests"] branch = true parallel = true -omit = [ - "src/amazon_bedrock_haystack/__about__.py", -] + [tool.coverage.paths] -amazon_bedrock_haystack = ["src/amazon_bedrock_haystack", "*/amazon_bedrock/src/amazon_bedrock_haystack"] -tests = ["tests", "*/amazon_bedrock_haystack/tests"] +amazon_bedrock_haystack = ["src/*"] +tests = ["tests"] [tool.coverage.report] exclude_lines = [ @@ -170,6 +172,7 @@ module = [ "transformers.*", "boto3.*", "haystack.*", + "haystack_integrations.*", "pytest.*", "numpy.*", ] diff --git a/integrations/amazon_bedrock/src/amazon_bedrock_haystack/generators/__init__.py b/integrations/amazon_bedrock/src/amazon_bedrock_haystack/generators/__init__.py deleted file mode 100644 index e873bc332..000000000 --- a/integrations/amazon_bedrock/src/amazon_bedrock_haystack/generators/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# SPDX-FileCopyrightText: 2023-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 diff --git a/integrations/amazon_bedrock/src/amazon_bedrock_haystack/__init__.py b/integrations/amazon_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/__init__.py similarity index 63% rename from integrations/amazon_bedrock/src/amazon_bedrock_haystack/__init__.py rename to integrations/amazon_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/__init__.py index 3e05179c0..236347b61 100644 --- a/integrations/amazon_bedrock/src/amazon_bedrock_haystack/__init__.py +++ b/integrations/amazon_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/__init__.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: 2023-present deepset GmbH # # SPDX-License-Identifier: Apache-2.0 -from amazon_bedrock_haystack.generators.amazon_bedrock import AmazonBedrockGenerator +from .generator import AmazonBedrockGenerator __all__ = ["AmazonBedrockGenerator"] diff --git a/integrations/amazon_bedrock/src/amazon_bedrock_haystack/generators/amazon_bedrock_adapters.py b/integrations/amazon_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/amazon_bedrock_adapters.py similarity index 98% rename from integrations/amazon_bedrock/src/amazon_bedrock_haystack/generators/amazon_bedrock_adapters.py rename to integrations/amazon_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/amazon_bedrock_adapters.py index bec172867..b7e775cb8 100644 --- a/integrations/amazon_bedrock/src/amazon_bedrock_haystack/generators/amazon_bedrock_adapters.py +++ b/integrations/amazon_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/amazon_bedrock_adapters.py @@ -2,7 +2,7 @@ from abc import ABC, abstractmethod from typing import Any, Dict, List, Optional -from amazon_bedrock_haystack.generators.amazon_bedrock_handlers import TokenStreamingHandler +from .amazon_bedrock_handlers import TokenStreamingHandler class BedrockModelAdapter(ABC): diff --git a/integrations/amazon_bedrock/src/amazon_bedrock_haystack/generators/amazon_bedrock_handlers.py b/integrations/amazon_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/amazon_bedrock_handlers.py similarity index 100% rename from integrations/amazon_bedrock/src/amazon_bedrock_haystack/generators/amazon_bedrock_handlers.py rename to integrations/amazon_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/amazon_bedrock_handlers.py diff --git a/integrations/amazon_bedrock/src/amazon_bedrock_haystack/errors.py b/integrations/amazon_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/errors.py similarity index 100% rename from integrations/amazon_bedrock/src/amazon_bedrock_haystack/errors.py rename to integrations/amazon_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/errors.py diff --git a/integrations/amazon_bedrock/src/amazon_bedrock_haystack/generators/amazon_bedrock.py b/integrations/amazon_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/generator.py similarity index 98% rename from integrations/amazon_bedrock/src/amazon_bedrock_haystack/generators/amazon_bedrock.py rename to integrations/amazon_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/generator.py index dda84fe14..c79ef9de4 100644 --- a/integrations/amazon_bedrock/src/amazon_bedrock_haystack/generators/amazon_bedrock.py +++ b/integrations/amazon_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/generator.py @@ -7,12 +7,7 @@ from botocore.exceptions import BotoCoreError, ClientError from haystack import component, default_from_dict, default_to_dict -from amazon_bedrock_haystack.errors import ( - AmazonBedrockConfigurationError, - AmazonBedrockInferenceError, - AWSConfigurationError, -) -from amazon_bedrock_haystack.generators.amazon_bedrock_adapters import ( +from .amazon_bedrock_adapters import ( AI21LabsJurassic2Adapter, AmazonTitanAdapter, AnthropicClaudeAdapter, @@ -20,11 +15,16 @@ CohereCommandAdapter, MetaLlama2ChatAdapter, ) -from amazon_bedrock_haystack.generators.amazon_bedrock_handlers import ( +from .amazon_bedrock_handlers import ( DefaultPromptHandler, DefaultTokenStreamingHandler, TokenStreamingHandler, ) +from .errors import ( + AmazonBedrockConfigurationError, + AmazonBedrockInferenceError, + AWSConfigurationError, +) logger = logging.getLogger(__name__) diff --git a/integrations/amazon_bedrock/tests/test_amazon_bedrock.py b/integrations/amazon_bedrock/tests/test_amazon_bedrock.py index a05c95ba3..c6bb0add4 100644 --- a/integrations/amazon_bedrock/tests/test_amazon_bedrock.py +++ b/integrations/amazon_bedrock/tests/test_amazon_bedrock.py @@ -4,9 +4,8 @@ import pytest from botocore.exceptions import BotoCoreError -from amazon_bedrock_haystack.errors import AmazonBedrockConfigurationError -from amazon_bedrock_haystack.generators.amazon_bedrock import AmazonBedrockGenerator -from amazon_bedrock_haystack.generators.amazon_bedrock_adapters import ( +from haystack_integrations.components.generators.amazon_bedrock import AmazonBedrockGenerator +from haystack_integrations.components.generators.amazon_bedrock.amazon_bedrock_adapters import ( AI21LabsJurassic2Adapter, AmazonTitanAdapter, AnthropicClaudeAdapter, @@ -14,6 +13,7 @@ CohereCommandAdapter, MetaLlama2ChatAdapter, ) +from haystack_integrations.components.generators.amazon_bedrock.errors import AmazonBedrockConfigurationError @pytest.fixture @@ -34,7 +34,7 @@ def mock_boto3_session(): @pytest.fixture def mock_prompt_handler(): with patch( - "amazon_bedrock_haystack.generators.amazon_bedrock_handlers.DefaultPromptHandler" + "haystack_integrations.components.generators.amazon_bedrock.amazon_bedrock_handlers.DefaultPromptHandler" ) as mock_prompt_handler: yield mock_prompt_handler @@ -55,7 +55,7 @@ def test_to_dict(mock_auto_tokenizer, mock_boto3_session): ) expected_dict = { - "type": "amazon_bedrock_haystack.generators.amazon_bedrock.AmazonBedrockGenerator", + "type": "haystack_integrations.components.generators.amazon_bedrock.generator.AmazonBedrockGenerator", "init_parameters": { "model": "anthropic.claude-v2", "max_length": 99, @@ -72,7 +72,7 @@ def test_from_dict(mock_auto_tokenizer, mock_boto3_session): """ generator = AmazonBedrockGenerator.from_dict( { - "type": "amazon_bedrock_haystack.generators.amazon_bedrock.AmazonBedrockGenerator", + "type": "haystack_integrations.components.generators.amazon_bedrock.generator.AmazonBedrockGenerator", "init_parameters": { "model": "anthropic.claude-v2", "max_length": 99, @@ -235,7 +235,7 @@ def test_supports_for_valid_aws_configuration(): # Patch the class method to return the mock session with patch( - "amazon_bedrock_haystack.generators.amazon_bedrock.AmazonBedrockGenerator.get_aws_session", + "haystack_integrations.components.generators.amazon_bedrock.AmazonBedrockGenerator.get_aws_session", return_value=mock_session, ): supported = AmazonBedrockGenerator.supports( @@ -266,7 +266,7 @@ def test_supports_for_invalid_bedrock_config(): # Patch the class method to return the mock session with patch( - "amazon_bedrock_haystack.generators.amazon_bedrock.AmazonBedrockGenerator.get_aws_session", + "haystack_integrations.components.generators.amazon_bedrock.AmazonBedrockGenerator.get_aws_session", return_value=mock_session, ), pytest.raises(AmazonBedrockConfigurationError, match="Could not connect to Amazon Bedrock."): AmazonBedrockGenerator.supports( @@ -282,7 +282,7 @@ def test_supports_for_invalid_bedrock_config_error_on_list_models(): # Patch the class method to return the mock session with patch( - "amazon_bedrock_haystack.generators.amazon_bedrock.AmazonBedrockGenerator.get_aws_session", + "haystack_integrations.components.generators.amazon_bedrock.AmazonBedrockGenerator.get_aws_session", return_value=mock_session, ), pytest.raises(AmazonBedrockConfigurationError, match="Could not connect to Amazon Bedrock."): AmazonBedrockGenerator.supports( @@ -314,7 +314,7 @@ def test_supports_with_stream_true_for_model_that_supports_streaming(): # Patch the class method to return the mock session with patch( - "amazon_bedrock_haystack.generators.amazon_bedrock.AmazonBedrockGenerator.get_aws_session", + "haystack_integrations.components.generators.amazon_bedrock.AmazonBedrockGenerator.get_aws_session", return_value=mock_session, ): supported = AmazonBedrockGenerator.supports( @@ -335,7 +335,7 @@ def test_supports_with_stream_true_for_model_that_does_not_support_streaming(): # Patch the class method to return the mock session with patch( - "amazon_bedrock_haystack.generators.amazon_bedrock.AmazonBedrockGenerator.get_aws_session", + "haystack_integrations.components.generators.amazon_bedrock.AmazonBedrockGenerator.get_aws_session", return_value=mock_session, ), pytest.raises( AmazonBedrockConfigurationError,