Merge remote-tracking branch 'origin/main' into add_pinecone

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,19 @@
+---
+name: Bug report
+about: Report a bug for an integration
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior. Feel free to link a Colab we can run to investigate the issue.
+
+**Describe your environment (please complete the following information):**
+ - OS: [e.g. iOS]
+ - Haystack version:
+ - Integration version:

.github/ISSUE_TEMPLATE/feature-request-for-existing-integrations.md

@@ -0,0 +1,20 @@
+---
+name: Feature request for existing integrations
+about: Suggest an idea for an integration
+title: ''
+labels: feature request
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/new-integration-proposal.md

@@ -0,0 +1,31 @@
+---
+name: New Integration Proposal
+about: Track the creation process for a new integration
+title: ''
+labels: New integration request
+assignees: ''
+
+---
+
+## Summary and motivation
+
+Briefly explain the feature request: why do we need this feature? What use cases does it support?
+
+## Alternatives
+
+A clear and concise description of any alternative solutions or features you've considered.
+
+## Detailed design
+
+Explain the design in enough detail for somebody familiar with Haystack to understand, and for somebody familiar with the implementation to implement. Get into specifics and corner-cases, and include examples of how the feature is used. Also, if there's any new terminology involved, define it here.
+
+## Checklist
+
+If the feature request is accepted, ensure the following checklist is complete before closing the issue.
+
+- [ ] The package has been released on PyPI
+- [ ] There is a Github workflow running the tests for the integration nightly and at every PR
+- [ ] A label named like `integration:<your integration name>` has been added to this repo
+- [ ] The [labeler.yml](https://github.com/deepset-ai/haystack-core-integrations/blob/main/.github/labeler.yml) file has been updated
+- [ ] An integration tile has been added to https://github.com/deepset-ai/haystack-integrations
+- [ ] The integration has been listed in the [Inventory section](https://github.com/deepset-ai/haystack-core-integrations#inventory) of this repo README

.github/labeler.yml

@@ -1,10 +1,16 @@
 # Integrations
-integration:instructor-embedders:
-- components/instructor-embedders/**/*
 integration:chroma:
-- document_stores/chroma/**/*
+- integrations/chroma/**/*
 integration:elasticsearch:
-- document_stores/elasticsearch/**/*
+- integrations/elasticsearch/**/*
+integration:gradient:
+- integrations/gradient/**/*
+integration:instructor-embedders:
+- integrations/instructor-embedders/**/*
+integration:opensearch:
+- integrations/opensearch/**/*
+integration:unstructured-fileconverter:
+- integrations/unstructured/fileconverter/**/*
 # Topics
 topic:CI:
 - .github/*

.github/workflows/document_stores_chroma.yml → .github/workflows/chroma.yml

@@ -1,21 +1,21 @@
 # This workflow comes from https://github.com/ofek/hatch-mypyc
 # https://github.com/ofek/hatch-mypyc/blob/5a198c0ba8660494d02716cfc9d79ce4adfb1442/.github/workflows/test.yml
-name: Test / Document Stores / chroma
+name: Test / chroma
 
 on:
   schedule:
     - cron: "0 0 * * *"
   pull_request:
     paths:
-      - 'document_stores/chroma/**'
-      - '.github/workflows/document_stores_chroma.yml'
+      - 'integrations/chroma/**'
+      - '.github/workflows/chroma.yml'
 
 defaults:
   run:
-    working-directory: document_stores/chroma
+    working-directory: integrations/chroma
 
 concurrency:
-  group: document_stores_chroma-${{ github.head_ref }}
+  group: chroma-${{ github.head_ref }}
   cancel-in-progress: true
 
 env:

.github/workflows/cohere.yml

@@ -0,0 +1,56 @@
+# This workflow comes from https://github.com/ofek/hatch-mypyc
+# https://github.com/ofek/hatch-mypyc/blob/5a198c0ba8660494d02716cfc9d79ce4adfb1442/.github/workflows/test.yml
+name: Test / cohere
+
+on:
+  schedule:
+    - cron: "0 0 * * *"
+  pull_request:
+    paths:
+      - 'integrations/cohere/**'
+      - '.github/workflows/cohere.yml'
+
+defaults:
+  run:
+    working-directory: integrations/cohere
+
+concurrency:
+  group: cohere-${{ github.head_ref }}
+  cancel-in-progress: true
+
+env:
+  PYTHONUNBUFFERED: "1"
+  FORCE_COLOR: "1"
+
+jobs:
+  run:
+    name: Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: ['3.9', '3.10']
+
+    steps:
+    - name: Support longpaths
+      if: matrix.os == 'windows-latest'
+      working-directory: .
+      run: git config --system core.longpaths true
+
+    - uses: actions/checkout@v4
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install Hatch
+      run: pip install --upgrade hatch
+
+    - name: Lint
+      if: matrix.python-version == '3.9' && runner.os == 'Linux'
+      run: hatch run lint:all
+
+    - name: Run tests
+      run: hatch run cov

.github/workflows/document_stores_elasticsearch.yml → .github/workflows/elasticsearch.yml

@@ -1,17 +1,17 @@
 # This workflow comes from https://github.com/ofek/hatch-mypyc
 # https://github.com/ofek/hatch-mypyc/blob/5a198c0ba8660494d02716cfc9d79ce4adfb1442/.github/workflows/test.yml
-name: Test / Document Stores / elasticsearch
+name: Test / elasticsearch
 
 on:
   schedule:
     - cron: "0 0 * * *"
   pull_request:
     paths:
-      - "document_stores/elasticsearch/**"
-      - ".github/workflows/document_stores_elasticsearch.yml"
+      - "integrations/elasticsearch/**"
+      - ".github/workflows/elasticsearch.yml"
 
 concurrency:
-  group: document_stores_elasticsearch-${{ github.head_ref }}
+  group: elasticsearch-${{ github.head_ref }}
   cancel-in-progress: true
 
 env:
@@ -40,14 +40,14 @@ jobs:
         run: pip install --upgrade hatch
 
       - name: Lint
-        working-directory: document_stores/elasticsearch
+        working-directory: integrations/elasticsearch
         if: matrix.python-version == '3.9'
         run: hatch run lint:all
 
       - name: Run ElasticSearch container
-        working-directory: document_stores/elasticsearch
+        working-directory: integrations/elasticsearch
         run: docker-compose up -d
 
       - name: Run tests
-        working-directory: document_stores/elasticsearch
+        working-directory: integrations/elasticsearch
         run: hatch run cov

.github/workflows/gradient.yml

@@ -0,0 +1,56 @@
+# This workflow comes from https://github.com/ofek/hatch-mypyc
+# https://github.com/ofek/hatch-mypyc/blob/5a198c0ba8660494d02716cfc9d79ce4adfb1442/.github/workflows/test.yml
+name: Test / gradient
+
+on:
+  schedule:
+    - cron: "0 0 * * *"
+  pull_request:
+    paths:
+      - 'integrations/gradient/**'
+      - '.github/workflows/gradient.yml'
+
+defaults:
+  run:
+    working-directory: integrations/gradient
+
+concurrency:
+  group: gradient-${{ github.head_ref }}
+  cancel-in-progress: true
+
+env:
+  PYTHONUNBUFFERED: "1"
+  FORCE_COLOR: "1"
+
+jobs:
+  run:
+    name: Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: ['3.9', '3.10']
+
+    steps:
+    - name: Support longpaths
+      if: matrix.os == 'windows-latest'
+      working-directory: .
+      run: git config --system core.longpaths true
+
+    - uses: actions/checkout@v4
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install Hatch
+      run: pip install --upgrade hatch
+
+    - name: Lint
+      if: matrix.python-version == '3.9' && runner.os == 'Linux'
+      run: hatch run lint:all
+
+    - name: Run tests
+      run: hatch run cov

.github/workflows/components_instructor_embedders.yml → .github/workflows/instructor_embedders.yml

@@ -1,16 +1,16 @@
-name: Test / Components / instructor-embedders
+name: Test / instructor-embedders
 
 on:
   schedule:
     - cron: "0 0 * * *"
   pull_request:
     paths:
-      - 'components/instructor-embedders/**'
-      - '.github/workflows/components_instructor_embedders.yml'
+      - 'integrations/instructor-embedders/**'
+      - '.github/workflows/instructor_embedders.yml'
 
 defaults:
   run:
-    working-directory: components/instructor-embedders
+    working-directory: integrations/instructor-embedders
 
 jobs:
   test:
@@ -27,7 +27,7 @@ jobs:
     - name: Ruff
       uses: chartboost/ruff-action@v1
       with:
-        src: components/instructor-embedders
+        src: integrations/instructor-embedders
 
     - name: Install instructor-embedders
       run: |

.github/workflows/labeler.yml

@@ -0,0 +1,15 @@
+name: "Labeler"
+on:
+- pull_request_target
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  triage:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/labeler@v4
+      with:
+        repo-token: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/nodes_text2speech.yml

@@ -1,16 +1,14 @@
 name: Test / text2speech
 
 on:
-  schedule:
-    - cron: "0 0 * * *"
   pull_request:
     paths:
-      - 'nodes/text2speech/**'
+      - 'integrations/nodes/text2speech/**'
       - '.github/workflows/nodes_text2speech.yml'
 
 defaults:
   run:
-    working-directory: nodes/text2speech
+    working-directory: integrations/nodes/text2speech
 
 jobs:
   test:

.github/workflows/opensearch.yml

@@ -0,0 +1,53 @@
+# This workflow comes from https://github.com/ofek/hatch-mypyc
+# https://github.com/ofek/hatch-mypyc/blob/5a198c0ba8660494d02716cfc9d79ce4adfb1442/.github/workflows/test.yml
+name: Test / opensearch
+
+on:
+  schedule:
+    - cron: "0 0 * * *"
+  pull_request:
+    paths:
+      - "integrations/opensearch/**"
+      - ".github/workflows/opensearch.yml"
+
+concurrency:
+  group: opensearch-${{ github.head_ref }}
+  cancel-in-progress: true
+
+env:
+  PYTHONUNBUFFERED: "1"
+  FORCE_COLOR: "1"
+
+jobs:
+  run:
+    name: Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Hatch
+        run: pip install --upgrade hatch
+
+      - name: Lint
+        working-directory: integrations/opensearch
+        if: matrix.python-version == '3.9'
+        run: hatch run lint:all
+
+      - name: Run opensearch container
+        working-directory: integrations/opensearch
+        run: docker-compose up -d
+
+      - name: Run tests
+        working-directory: integrations/opensearch
+        run: hatch run cov