Merge branch 'deepset-ai:main' into chroma_persistent

deepset-ai · Feb 5, 2024 · 50c1bec · 50c1bec
2 parents ca446f3 + 6d18bc4
commit 50c1bec
Show file tree

Hide file tree

Showing 58 changed files with 2,255 additions and 118 deletions.
diff --git a/.github/ISSUE_TEMPLATE/breaking-change-proposal.md b/.github/ISSUE_TEMPLATE/breaking-change-proposal.md
@@ -15,9 +15,12 @@ Briefly explain how the change is breaking and why is needed.
 
 ```[tasklist]
 ### Tasks
-- [ ] The change is documented with docstrings and was merged in the `main` branch
-- [ ] Integration tile on https://github.com/deepset-ai/haystack-integrations was updated
+- [ ] The changes are merged in the `main` branch (Code + Docstrings)
+- [ ] New package version declares the breaking change
+- [ ] The package has been released on PyPI
 - [ ] Docs at https://docs.haystack.deepset.ai/ were updated
+- [ ] Integration tile on https://github.com/deepset-ai/haystack-integrations was updated
 - [ ] Notebooks on https://github.com/deepset-ai/haystack-cookbook were updated (if needed)
-- [ ] New package version declares the breaking change and package has been released on PyPI
-```
+- [ ] Tutorials on https://github.com/deepset-ai/haystack-tutorials were updated (if needed)
+- [ ] Articles on https://github.com/deepset-ai/haystack-home/tree/main/content were updated (if needed)
+```
diff --git a/.github/labeler.yml b/.github/labeler.yml
@@ -4,6 +4,11 @@ integration:amazon-bedrock:
       - any-glob-to-any-file: "integrations/amazon_bedrock/**/*"
       - any-glob-to-any-file: ".github/workflows/amazon_bedrock.yml"
 
+integration:amazon-sagemaker:
+  - changed-files:
+      - any-glob-to-any-file: "integrations/amazon_sagemaker/**/*"
+      - any-glob-to-any-file: ".github/workflows/amazon_sagemaker.yml"
+
 integration:astra:
   - changed-files:
       - any-glob-to-any-file: "integrations/astra/**/*"
@@ -64,6 +69,11 @@ integration:opensearch:
       - any-glob-to-any-file: "integrations/opensearch/**/*"
       - any-glob-to-any-file: ".github/workflows/opensearch.yml"
 
+integration:pgvector:
+  - changed-files:
+      - any-glob-to-any-file: "integrations/pgvector/**/*"
+      - any-glob-to-any-file: ".github/workflows/pgvector.yml"      
+
 integration:pinecone:
   - changed-files:
       - any-glob-to-any-file: "integrations/pinecone/**/*"
@@ -76,8 +86,8 @@ integration:qdrant:
 
 integration:unstructured-fileconverter:
   - changed-files:
-      - any-glob-to-any-file: "integrations/unstructured/fileconverter/**/*"
-      - any-glob-to-any-file: ".github/workflows/unstructured_fileconverter.yml"
+      - any-glob-to-any-file: "integrations/unstructured/**/*"
+      - any-glob-to-any-file: ".github/workflows/unstructured.yml"
 
 integration:uptrain:
   - changed-files:

diff --git a/.github/workflows/amazon_sagemaker.yml b/.github/workflows/amazon_sagemaker.yml
@@ -0,0 +1,56 @@
+# This workflow comes from https://github.com/ofek/hatch-mypyc
+# https://github.com/ofek/hatch-mypyc/blob/5a198c0ba8660494d02716cfc9d79ce4adfb1442/.github/workflows/test.yml
+name: Test / amazon-sagemaker
+
+on:
+  schedule:
+    - cron: "0 0 * * *"
+  pull_request:
+    paths:
+      - "integrations/amazon_sagemaker/**"
+      - ".github/workflows/amazon_sagemaker.yml"
+
+defaults:
+  run:
+    working-directory: integrations/amazon_sagemaker
+
+concurrency:
+  group: amazon-sagemaker-${{ github.head_ref }}
+  cancel-in-progress: true
+
+env:
+  PYTHONUNBUFFERED: "1"
+  FORCE_COLOR: "1"
+
+jobs:
+  run:
+    name: Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: ["3.9", "3.10"]
+
+    steps:
+      - name: Support longpaths
+        if: matrix.os == 'windows-latest'
+        working-directory: .
+        run: git config --system core.longpaths true
+
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Hatch
+        run: pip install --upgrade hatch
+
+      - name: Lint
+        if: matrix.python-version == '3.9' && runner.os == 'Linux'
+        run: hatch run lint:all
+
+      - name: Run tests
+        run: hatch run cov
diff --git a/.github/workflows/astra.yml b/.github/workflows/astra.yml
@@ -53,6 +53,10 @@ jobs:
       if: matrix.python-version == '3.9' && runner.os == 'Linux'
       run: hatch run lint:all
 
+    - name: Generate docs
+      if: matrix.python-version == '3.9' && runner.os == 'Linux'
+      run: hatch run docs
+
     - name: Run tests
       env:
         ASTRA_DB_APPLICATION_TOKEN: ${{ secrets.ASTRA_DB_APPLICATION_TOKEN }}

diff --git a/.github/workflows/cohere.yml b/.github/workflows/cohere.yml
@@ -7,8 +7,8 @@ on:
     - cron: "0 0 * * *"
   pull_request:
     paths:
-      - 'integrations/cohere/**'
-      - '.github/workflows/cohere.yml'
+      - "integrations/cohere/**"
+      - ".github/workflows/cohere.yml"
 
 defaults:
   run:
@@ -21,6 +21,7 @@ concurrency:
 env:
   PYTHONUNBUFFERED: "1"
   FORCE_COLOR: "1"
+  COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
 
 jobs:
   run:
@@ -30,27 +31,27 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ['3.9', '3.10']
+        python-version: ["3.9", "3.10"]
 
     steps:
-    - name: Support longpaths
-      if: matrix.os == 'windows-latest'
-      working-directory: .
-      run: git config --system core.longpaths true
+      - name: Support longpaths
+        if: matrix.os == 'windows-latest'
+        working-directory: .
+        run: git config --system core.longpaths true
 
-    - uses: actions/checkout@v4
+      - uses: actions/checkout@v4
 
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python-version }}
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
 
-    - name: Install Hatch
-      run: pip install --upgrade hatch
+      - name: Install Hatch
+        run: pip install --upgrade hatch
 
-    - name: Lint
-      if: matrix.python-version == '3.9' && runner.os == 'Linux'
-      run: hatch run lint:all
+      - name: Lint
+        if: matrix.python-version == '3.9' && runner.os == 'Linux'
+        run: hatch run lint:all
 
-    - name: Run tests
-      run: hatch run cov
+      - name: Run tests
+        run: hatch run cov
diff --git a/.github/workflows/elasticsearch.yml b/.github/workflows/elasticsearch.yml
@@ -10,6 +10,10 @@ on:
       - "integrations/elasticsearch/**"
       - ".github/workflows/elasticsearch.yml"
 
+defaults:
+  run:
+    working-directory: integrations/elasticsearch
+
 concurrency:
   group: elasticsearch-${{ github.head_ref }}
   cancel-in-progress: true
@@ -40,14 +44,15 @@ jobs:
         run: pip install --upgrade hatch
 
       - name: Lint
-        working-directory: integrations/elasticsearch
         if: matrix.python-version == '3.9'
         run: hatch run lint:all
 
       - name: Run ElasticSearch container
-        working-directory: integrations/elasticsearch
         run: docker-compose up -d
 
+      - name: Generate docs
+        if: matrix.python-version == '3.9' && runner.os == 'Linux'
+        run: hatch run docs
+
       - name: Run tests
-        working-directory: integrations/elasticsearch
         run: hatch run cov
diff --git a/.github/workflows/ollama.yml b/.github/workflows/ollama.yml
@@ -54,6 +54,11 @@ jobs:
       - name: Pull the LLM in the Ollama service
         run: docker exec ollama ollama pull ${{ env.LLM_FOR_TESTS }}        
 
+      - name: Generate docs
+        working-directory: integrations/ollama
+        if: matrix.python-version == '3.9' && runner.os == 'Linux'
+        run: hatch run docs
+
       - name: Run tests
         working-directory: integrations/ollama
         run: hatch run cov
diff --git a/.github/workflows/opensearch.yml b/.github/workflows/opensearch.yml
@@ -18,6 +18,10 @@ env:
   PYTHONUNBUFFERED: "1"
   FORCE_COLOR: "1"
 
+defaults:
+  run:
+    working-directory: integrations/opensearch  
+
 jobs:
   run:
     name: Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }}
@@ -40,14 +44,16 @@ jobs:
         run: pip install --upgrade hatch
 
       - name: Lint
-        working-directory: integrations/opensearch
         if: matrix.python-version == '3.9'
         run: hatch run lint:all
 
       - name: Run opensearch container
-        working-directory: integrations/opensearch
         run: docker-compose up -d
 
+      - name: Generate docs
+        if: matrix.python-version == '3.9' && runner.os == 'Linux'
+        run: hatch run docs        
+
       - name: Run tests
         working-directory: integrations/opensearch
         run: hatch run cov
diff --git a/.github/workflows/pgvector.yml b/.github/workflows/pgvector.yml
@@ -18,6 +18,10 @@ env:
   PYTHONUNBUFFERED: "1"
   FORCE_COLOR: "1"
 
+defaults:
+  run:
+    working-directory: integrations/pgvector  
+
 jobs:
   run:
     name: Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }}
@@ -49,10 +53,12 @@ jobs:
         run: pip install --upgrade hatch
 
       - name: Lint
-        working-directory: integrations/pgvector
         if: matrix.python-version == '3.9'
-        run: hatch run lint:all  
+        run: hatch run lint:all
+
+      - name: Generate docs
+        if: matrix.python-version == '3.9' && runner.os == 'Linux'
+        run: hatch run docs          
 
       - name: Run tests
-        working-directory: integrations/pgvector
         run: hatch run cov
diff --git a/README.md b/README.md
@@ -80,3 +80,4 @@ deepset-haystack
 | [qdrant-haystack](integrations/qdrant/)                             | Document Store      | [![PyPI - Version](https://img.shields.io/pypi/v/qdrant-haystack.svg?color=orange)](https://pypi.org/project/qdrant-haystack)                            | [![Test / qdrant](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/qdrant.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/qdrant.yml)                                           |
 | [unstructured-fileconverter-haystack](integrations/unstructured/)   | File converter      | [![PyPI - Version](https://img.shields.io/pypi/v/unstructured-fileconverter-haystack.svg)](https://pypi.org/project/unstructured-fileconverter-haystack) | [![Test / unstructured](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/unstructured.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/unstructured.yml)                         |
 | [uptrain-haystack](integrations/uptrain/)                           | Evaluator           | [![PyPI - Version](https://img.shields.io/pypi/v/uptrain-haystack.svg)](https://pypi.org/project/uptrain-haystack)                                       | [![Test / uptrain](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/uptrain.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/uptrain.yml)                                        |
+| [amazon-sagemaker-haystack](integrations/amazon_sagemaker/)         | Generator           | [![PyPI - Version](https://img.shields.io/pypi/v/amazon-sagemaker-haystack.svg)](https://pypi.org/project/amazon-sagemaker-haystack)                     | [![Test / amazon_sagemaker](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/amazon_sagemaker.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/amazon_sagemaker.yml)             |
diff --git a/integrations/amazon_bedrock/README.md b/integrations/amazon_bedrock/README.md
@@ -8,6 +8,7 @@
 **Table of Contents**
 
 - [Installation](#installation)
+- [Contributing](#contributing)
 - [License](#license)
 
 ## Installation
@@ -16,6 +17,24 @@
 pip install amazon-bedrock-haystack
 ```
 
+## Contributing
+
+`hatch` is the best way to interact with this project, to install it:
+```sh
+pip install hatch
+```
+
+With `hatch` installed, to run all the tests:
+```
+hatch run test
+```
+> Note: there are no integration tests for this project.
+
+To run the linters `ruff` and `mypy`:
+```
+hatch run lint:all
+```
+
 ## License
 
 `amazon-bedrock-haystack` is distributed under the terms of the [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) license.
diff --git a/integrations/amazon_bedrock/pyproject.toml b/integrations/amazon_bedrock/pyproject.toml
@@ -35,6 +35,9 @@ Documentation = "https://github.com/deepset-ai/haystack-core-integrations/tree/m
 Issues = "https://github.com/deepset-ai/haystack-core-integrations/issues"
 Source = "https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/amazon_bedrock"
 
+[tool.hatch.build.targets.wheel]
+packages = ["src/haystack_integrations"]
+
 [tool.hatch.version]
 source = "vcs"
 tag-pattern = 'integrations\/amazon_bedrock-v(?P<version>.*)'
@@ -71,7 +74,8 @@ dependencies = [
   "ruff>=0.0.243",
 ]
 [tool.hatch.envs.lint.scripts]
-typing = "mypy --install-types --non-interactive {args:src/amazon_bedrock_haystack tests}"
+typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}"
+
 style = [
   "ruff {args:.}",
   "black --check --diff {args:.}",
@@ -136,26 +140,24 @@ unfixable = [
 ]
 
 [tool.ruff.isort]
-known-first-party = ["amazon_bedrock_haystack"]
+known-first-party = ["haystack_integrations"]
 
 [tool.ruff.flake8-tidy-imports]
-ban-relative-imports = "all"
+ban-relative-imports = "parents"
 
 [tool.ruff.per-file-ignores]
 # Tests can use magic values, assertions, and relative imports
 "tests/**/*" = ["PLR2004", "S101", "TID252"]
 
 [tool.coverage.run]
-source_pkgs = ["amazon_bedrock_haystack", "tests"]
+source_pkgs = ["src", "tests"]
 branch = true
 parallel = true
-omit = [
-  "src/amazon_bedrock_haystack/__about__.py",
-]
+
 
 [tool.coverage.paths]
-amazon_bedrock_haystack = ["src/amazon_bedrock_haystack", "*/amazon_bedrock/src/amazon_bedrock_haystack"]
-tests = ["tests", "*/amazon_bedrock_haystack/tests"]
+amazon_bedrock_haystack = ["src/*"]
+tests = ["tests"]
 
 [tool.coverage.report]
 exclude_lines = [
@@ -170,6 +172,7 @@ module = [
   "transformers.*",
   "boto3.*",
   "haystack.*",
+  "haystack_integrations.*",
   "pytest.*",
   "numpy.*",
 ]

diff --git a/...k/src/amazon_bedrock_haystack/__init__.py → ...nts/generators/amazon_bedrock/__init__.py b/...k/src/amazon_bedrock_haystack/__init__.py → ...nts/generators/amazon_bedrock/__init__.py
@@ -1,6 +1,6 @@
 # SPDX-FileCopyrightText: 2023-present deepset GmbH <[email protected]>
 #
 # SPDX-License-Identifier: Apache-2.0
-from amazon_bedrock_haystack.generators.amazon_bedrock import AmazonBedrockGenerator
+from .generator import AmazonBedrockGenerator
 
 __all__ = ["AmazonBedrockGenerator"]