Merge branch 'main' into feat/unstructured_meta_field

deepset-ai · Feb 4, 2024 · 0199e3c · 0199e3c
2 parents dd40d0a + f7678e1
commit 0199e3c
Show file tree

Hide file tree

Showing 164 changed files with 5,473 additions and 552 deletions.
diff --git a/...TEMPLATE/new-integration-proposal copy.md → ...SSUE_TEMPLATE/breaking-change-proposal.md b/...TEMPLATE/new-integration-proposal copy.md → ...SSUE_TEMPLATE/breaking-change-proposal.md
@@ -13,8 +13,14 @@ Briefly explain how the change is breaking and why is needed.
 
 ## Checklist
 
-- [ ] The change is documented with docstrings and was merged in the `main` branch
-- [ ] Integration tile on https://github.com/deepset-ai/haystack-integrations was updated
+```[tasklist]
+### Tasks
+- [ ] The changes are merged in the `main` branch (Code + Docstrings)
+- [ ] New package version declares the breaking change
+- [ ] The package has been released on PyPI
 - [ ] Docs at https://docs.haystack.deepset.ai/ were updated
+- [ ] Integration tile on https://github.com/deepset-ai/haystack-integrations was updated
 - [ ] Notebooks on https://github.com/deepset-ai/haystack-cookbook were updated (if needed)
-- [ ] New package version declares the breaking change and package has been released on PyPI
+- [ ] Tutorials on https://github.com/deepset-ai/haystack-tutorials were updated (if needed)
+- [ ] Articles on https://github.com/deepset-ai/haystack-home/tree/main/content were updated (if needed)
+```
diff --git a/.github/ISSUE_TEMPLATE/new-integration-proposal.md b/.github/ISSUE_TEMPLATE/new-integration-proposal.md
@@ -20,7 +20,8 @@ Also, if there's any new terminology involved, define it here.
 ## Checklist
 
 If the request is accepted, ensure the following checklist is complete before closing this issue.
-
+```[tasklist]
+### Tasks
 - [ ] The code is documented with docstrings and was merged in the `main` branch
 - [ ] Docs are published at https://docs.haystack.deepset.ai/
 - [ ] There is a Github workflow running the tests for the integration nightly and at every PR
@@ -31,3 +32,4 @@ If the request is accepted, ensure the following checklist is complete before cl
 - [ ] The integration has been listed in the [Inventory section](https://github.com/deepset-ai/haystack-core-integrations#inventory) of this repo README
 - [ ] There is an example available to demonstrate the feature
 - [ ] The feature was announced through social media
+```
diff --git a/.github/labeler.yml b/.github/labeler.yml
@@ -4,6 +4,11 @@ integration:amazon-bedrock:
       - any-glob-to-any-file: "integrations/amazon_bedrock/**/*"
       - any-glob-to-any-file: ".github/workflows/amazon_bedrock.yml"
 
+integration:amazon-sagemaker:
+  - changed-files:
+      - any-glob-to-any-file: "integrations/amazon_sagemaker/**/*"
+      - any-glob-to-any-file: ".github/workflows/amazon_sagemaker.yml"
+
 integration:astra:
   - changed-files:
       - any-glob-to-any-file: "integrations/astra/**/*"
@@ -64,6 +69,11 @@ integration:opensearch:
       - any-glob-to-any-file: "integrations/opensearch/**/*"
       - any-glob-to-any-file: ".github/workflows/opensearch.yml"
 
+integration:pgvector:
+  - changed-files:
+      - any-glob-to-any-file: "integrations/pgvector/**/*"
+      - any-glob-to-any-file: ".github/workflows/pgvector.yml"      
+
 integration:pinecone:
   - changed-files:
       - any-glob-to-any-file: "integrations/pinecone/**/*"
@@ -76,8 +86,13 @@ integration:qdrant:
 
 integration:unstructured-fileconverter:
   - changed-files:
-      - any-glob-to-any-file: "integrations/unstructured/fileconverter/**/*"
-      - any-glob-to-any-file: ".github/workflows/unstructured_fileconverter.yml"
+      - any-glob-to-any-file: "integrations/unstructured/**/*"
+      - any-glob-to-any-file: ".github/workflows/unstructured.yml"
+
+integration:uptrain:
+  - changed-files:
+      - any-glob-to-any-file: "integrations/uptrain/**/*"
+      - any-glob-to-any-file: ".github/workflows/uptrain.yml"
 
 integration:weaviate:
   - changed-files:

diff --git a/.github/workflows/amazon_sagemaker.yml b/.github/workflows/amazon_sagemaker.yml
@@ -0,0 +1,56 @@
+# This workflow comes from https://github.com/ofek/hatch-mypyc
+# https://github.com/ofek/hatch-mypyc/blob/5a198c0ba8660494d02716cfc9d79ce4adfb1442/.github/workflows/test.yml
+name: Test / amazon-sagemaker
+
+on:
+  schedule:
+    - cron: "0 0 * * *"
+  pull_request:
+    paths:
+      - "integrations/amazon_sagemaker/**"
+      - ".github/workflows/amazon_sagemaker.yml"
+
+defaults:
+  run:
+    working-directory: integrations/amazon_sagemaker
+
+concurrency:
+  group: amazon-sagemaker-${{ github.head_ref }}
+  cancel-in-progress: true
+
+env:
+  PYTHONUNBUFFERED: "1"
+  FORCE_COLOR: "1"
+
+jobs:
+  run:
+    name: Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: ["3.9", "3.10"]
+
+    steps:
+      - name: Support longpaths
+        if: matrix.os == 'windows-latest'
+        working-directory: .
+        run: git config --system core.longpaths true
+
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Hatch
+        run: pip install --upgrade hatch
+
+      - name: Lint
+        if: matrix.python-version == '3.9' && runner.os == 'Linux'
+        run: hatch run lint:all
+
+      - name: Run tests
+        run: hatch run cov
diff --git a/.github/workflows/astra.yml b/.github/workflows/astra.yml
@@ -53,6 +53,10 @@ jobs:
       if: matrix.python-version == '3.9' && runner.os == 'Linux'
       run: hatch run lint:all
 
+    - name: Generate docs
+      if: matrix.python-version == '3.9' && runner.os == 'Linux'
+      run: hatch run docs
+
     - name: Run tests
       env:
         ASTRA_DB_APPLICATION_TOKEN: ${{ secrets.ASTRA_DB_APPLICATION_TOKEN }}

diff --git a/.github/workflows/chroma.yml b/.github/workflows/chroma.yml
@@ -52,5 +52,9 @@ jobs:
       if: matrix.python-version == '3.9' && runner.os == 'Linux'
       run: hatch run lint:all
 
+    - name: Generate docs
+      if: matrix.python-version == '3.9' && runner.os == 'Linux'
+      run: hatch run docs
+
     - name: Run tests
-      run: hatch run cov
+      run: hatch run cov
diff --git a/.github/workflows/cohere.yml b/.github/workflows/cohere.yml
@@ -52,5 +52,9 @@ jobs:
       if: matrix.python-version == '3.9' && runner.os == 'Linux'
       run: hatch run lint:all
 
+    - name: Generate docs
+      if: matrix.python-version == '3.9' && runner.os == 'Linux'
+      run: hatch run docs
+
     - name: Run tests
       run: hatch run cov
diff --git a/.github/workflows/elasticsearch.yml b/.github/workflows/elasticsearch.yml
@@ -10,6 +10,10 @@ on:
       - "integrations/elasticsearch/**"
       - ".github/workflows/elasticsearch.yml"
 
+defaults:
+  run:
+    working-directory: integrations/elasticsearch
+
 concurrency:
   group: elasticsearch-${{ github.head_ref }}
   cancel-in-progress: true
@@ -40,14 +44,15 @@ jobs:
         run: pip install --upgrade hatch
 
       - name: Lint
-        working-directory: integrations/elasticsearch
         if: matrix.python-version == '3.9'
         run: hatch run lint:all
 
       - name: Run ElasticSearch container
-        working-directory: integrations/elasticsearch
         run: docker-compose up -d
 
+      - name: Generate docs
+        if: matrix.python-version == '3.9' && runner.os == 'Linux'
+        run: hatch run docs
+
       - name: Run tests
-        working-directory: integrations/elasticsearch
         run: hatch run cov
diff --git a/.github/workflows/ollama.yml b/.github/workflows/ollama.yml
@@ -54,6 +54,11 @@ jobs:
       - name: Pull the LLM in the Ollama service
         run: docker exec ollama ollama pull ${{ env.LLM_FOR_TESTS }}        
 
+      - name: Generate docs
+        working-directory: integrations/ollama
+        if: matrix.python-version == '3.9' && runner.os == 'Linux'
+        run: hatch run docs
+
       - name: Run tests
         working-directory: integrations/ollama
         run: hatch run cov
diff --git a/.github/workflows/opensearch.yml b/.github/workflows/opensearch.yml
@@ -18,6 +18,10 @@ env:
   PYTHONUNBUFFERED: "1"
   FORCE_COLOR: "1"
 
+defaults:
+  run:
+    working-directory: integrations/opensearch  
+
 jobs:
   run:
     name: Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }}
@@ -40,14 +44,16 @@ jobs:
         run: pip install --upgrade hatch
 
       - name: Lint
-        working-directory: integrations/opensearch
         if: matrix.python-version == '3.9'
         run: hatch run lint:all
 
       - name: Run opensearch container
-        working-directory: integrations/opensearch
         run: docker-compose up -d
 
+      - name: Generate docs
+        if: matrix.python-version == '3.9' && runner.os == 'Linux'
+        run: hatch run docs        
+
       - name: Run tests
         working-directory: integrations/opensearch
         run: hatch run cov
diff --git a/.github/workflows/pgvector.yml b/.github/workflows/pgvector.yml
@@ -0,0 +1,64 @@
+# This workflow comes from https://github.com/ofek/hatch-mypyc
+# https://github.com/ofek/hatch-mypyc/blob/5a198c0ba8660494d02716cfc9d79ce4adfb1442/.github/workflows/test.yml
+name: Test / pgvector
+
+on:
+  schedule:
+    - cron: "0 0 * * *"
+  pull_request:
+    paths:
+      - "integrations/pgvector/**"
+      - ".github/workflows/pgvector.yml"
+
+concurrency:
+  group: pgvector-${{ github.head_ref }}
+  cancel-in-progress: true
+
+env:
+  PYTHONUNBUFFERED: "1"
+  FORCE_COLOR: "1"
+
+defaults:
+  run:
+    working-directory: integrations/pgvector  
+
+jobs:
+  run:
+    name: Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.9","3.10","3.11"]
+    services:
+      pgvector:
+        image: ankane/pgvector:latest
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        ports:
+          - 5432:5432
+
+    steps:   
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Hatch
+        run: pip install --upgrade hatch
+
+      - name: Lint
+        if: matrix.python-version == '3.9'
+        run: hatch run lint:all
+
+      - name: Generate docs
+        if: matrix.python-version == '3.9' && runner.os == 'Linux'
+        run: hatch run docs          
+
+      - name: Run tests
+        run: hatch run cov
diff --git a/.github/workflows/uptrain.yml b/.github/workflows/uptrain.yml
@@ -0,0 +1,56 @@
+# This workflow comes from https://github.com/ofek/hatch-mypyc
+# https://github.com/ofek/hatch-mypyc/blob/5a198c0ba8660494d02716cfc9d79ce4adfb1442/.github/workflows/test.yml
+name: Test / uptrain
+
+on:
+  schedule:
+    - cron: "0 0 * * *"
+  pull_request:
+    paths:
+      - "integrations/uptrain/**"
+      - ".github/workflows/uptrain.yml"
+
+defaults:
+  run:
+    working-directory: integrations/uptrain
+
+concurrency:
+  group: uptrain-${{ github.head_ref }}
+  cancel-in-progress: true
+
+env:
+  PYTHONUNBUFFERED: "1"
+  FORCE_COLOR: "1"
+
+jobs:
+  run:
+    name: Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: ["3.9", "3.10"]
+
+    steps:
+      - name: Support longpaths
+        if: matrix.os == 'windows-latest'
+        working-directory: .
+        run: git config --system core.longpaths true
+
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Hatch
+        run: pip install --upgrade hatch
+
+      - name: Lint
+        if: matrix.python-version == '3.9' && runner.os == 'Linux'
+        run: hatch run lint:all
+
+      - name: Run tests
+        run: hatch run cov
diff --git a/.github/workflows/weaviate.yml b/.github/workflows/weaviate.yml
@@ -29,15 +29,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest]
         python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
-      - name: Support longpaths
-        if: matrix.os == 'windows-latest'
-        working-directory: .
-        run: git config --system core.longpaths true
-
       - uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
@@ -49,8 +44,10 @@ jobs:
         run: pip install --upgrade hatch
 
       - name: Lint
-        if: runner.os == 'Linux'
         run: hatch run lint:all
 
+      - name: Run Weaviate container
+        run: docker-compose up -d
+
       - name: Run tests
         run: hatch run cov
diff --git a/.gitignore b/.gitignore
@@ -131,3 +131,6 @@ dmypy.json
 
 # IDEs
 .vscode
+
+# Docs generation artifacts
+_readme_*.md
-Original file line number
+Diff line change
@@ Expand Up / @@ -131,3 +131,6 @@ dmypy.json @@
     # IDEs
     .vscode
+    # Docs generation artifacts
+    _readme_*.md