Skip to content

Commit

Permalink
Merge branch 'main' into fix/weaviate-schema-class-name
Browse files Browse the repository at this point in the history
  • Loading branch information
davidsbatista authored May 3, 2024
2 parents 8b73372 + ca87b0e commit 17f6fe2
Show file tree
Hide file tree
Showing 31 changed files with 921 additions and 72 deletions.
5 changes: 5 additions & 0 deletions .github/labeler.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,11 @@ integration:jina:
- any-glob-to-any-file: "integrations/jina/**/*"
- any-glob-to-any-file: ".github/workflows/jina.yml"

integration:langfuse:
- changed-files:
- any-glob-to-any-file: "integrations/langfuse/**/*"
- any-glob-to-any-file: ".github/workflows/langfuse.yml"

integration:llama_cpp:
- changed-files:
- any-glob-to-any-file: "integrations/llama_cpp/**/*"
Expand Down
81 changes: 81 additions & 0 deletions .github/workflows/langfuse.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
# This workflow comes from https://github.com/ofek/hatch-mypyc
# https://github.com/ofek/hatch-mypyc/blob/5a198c0ba8660494d02716cfc9d79ce4adfb1442/.github/workflows/test.yml
name: Test / langfuse

on:
schedule:
- cron: "0 0 * * *"
pull_request:
paths:
- "integrations/langfuse/**"
- ".github/workflows/langfuse.yml"

defaults:
run:
working-directory: integrations/langfuse

concurrency:
group: langfuse-${{ github.head_ref }}
cancel-in-progress: true

env:
PYTHONUNBUFFERED: "1"
FORCE_COLOR: "1"
LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }}
LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}

jobs:
run:
name: Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest, macos-latest]
python-version: ["3.9", "3.10"]

steps:
- name: Support longpaths
if: matrix.os == 'windows-latest'
working-directory: .
run: git config --system core.longpaths true

- uses: actions/checkout@v4

- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}

- name: Install Hatch
run: pip install --upgrade hatch

- name: Lint
if: matrix.python-version == '3.9' && runner.os == 'Linux'
run: hatch run lint:all

- name: Generate docs
if: matrix.python-version == '3.9' && runner.os == 'Linux'
run: hatch run docs

- name: Run tests
id: tests
run: hatch run cov

- name: Nightly - run unit tests with Haystack main branch
if: github.event_name == 'schedule'
id: nightly-haystack-main
run: |
hatch run pip install git+https://github.com/deepset-ai/haystack.git
hatch run test -m "not integration"
- name: Send event to Datadog for nightly failures
if: failure() && github.event_name == 'schedule'
uses: ./.github/actions/send_failure
with:
title: |
core-integrations failure:
${{ (steps.tests.conclusion == 'nightly-haystack-main') && 'nightly-haystack-main' || 'tests' }}
- ${{ github.workflow }}
api-key: ${{ secrets.CORE_DATADOG_API_KEY }}
7 changes: 4 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,21 +23,22 @@ Please check out our [Contribution Guidelines](CONTRIBUTING.md) for all the deta
## Inventory

| Package | Type | PyPi Package | Status |
| -------------------------------------------------------------------------------------------------------------- | ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|----------------------------------------------------------------------------------------------------------------|---------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| [amazon-bedrock-haystack](integrations/amazon-bedrock/) | Generator | [![PyPI - Version](https://img.shields.io/pypi/v/amazon-bedrock-haystack.svg)](https://pypi.org/project/amazon-bedrock-haystack) | [![Test / amazon_bedrock](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/amazon_bedrock.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/amazon_bedrock.yml) |
| [amazon-sagemaker-haystack](integrations/amazon_sagemaker/) | Generator | [![PyPI - Version](https://img.shields.io/pypi/v/amazon-sagemaker-haystack.svg)](https://pypi.org/project/amazon-sagemaker-haystack) | [![Test / amazon_sagemaker](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/amazon_sagemaker.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/amazon_sagemaker.yml) |
| [anthropic-haystack](integrations/anthropic/) | Generator | [![PyPI - Version](https://img.shields.io/pypi/v/anthropic-haystack.svg)](https://pypi.org/project/anthropic-haystack) | [![Test / anthropic](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/anthropic.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/anthropic.yml) |
| [astra-haystack](integrations/astra/) | Document Store | [![PyPI - Version](https://img.shields.io/pypi/v/astra-haystack.svg)](https://pypi.org/project/astra-haystack) | [![Test / astra](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/astra.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/astra.yml) |
| [chroma-haystack](integrations/chroma/) | Document Store | [![PyPI - Version](https://img.shields.io/pypi/v/chroma-haystack.svg)](https://pypi.org/project/chroma-haystack) | [![Test / chroma](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/chroma.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/chroma.yml) |
| [cohere-haystack](integrations/cohere/) | Embedder, Generator | [![PyPI - Version](https://img.shields.io/pypi/v/cohere-haystack.svg)](https://pypi.org/project/cohere-haystack) | [![Test / cohere](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/cohere.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/cohere.yml) |
| [cohere-haystack](integrations/cohere/) | Embedder, Generator, Ranker | [![PyPI - Version](https://img.shields.io/pypi/v/cohere-haystack.svg)](https://pypi.org/project/cohere-haystack) | [![Test / cohere](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/cohere.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/cohere.yml) |
| [deepeval-haystack](integrations/deepeval/) | Evaluator | [![PyPI - Version](https://img.shields.io/pypi/v/deepeval-haystack.svg)](https://pypi.org/project/deepeval-haystack) | [![Test / deepeval](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/deepeval.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/deepeval.yml) |
| [elasticsearch-haystack](integrations/elasticsearch/) | Document Store | [![PyPI - Version](https://img.shields.io/pypi/v/elasticsearch-haystack.svg)](https://pypi.org/project/elasticsearch-haystack) | [![Test / elasticsearch](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/elasticsearch.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/elasticsearch.yml) |
| [fastembed-haystack](integrations/fastembed/) | Embedder | [![PyPI - Version](https://img.shields.io/pypi/v/fastembed-haystack.svg)](https://pypi.org/project/fastembed-haystack/) | [![Test / fastembed](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/fastembed.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/fastembed.yml) |
| [google-ai-haystack](integrations/google_ai/) | Generator | [![PyPI - Version](https://img.shields.io/pypi/v/google-ai-haystack.svg)](https://pypi.org/project/google-ai-haystack) | [![Test / google-ai](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/google_ai.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/google_ai.yml) |
| [google-vertex-haystack](integrations/google_vertex/) | Generator | [![PyPI - Version](https://img.shields.io/pypi/v/google-vertex-haystack.svg)](https://pypi.org/project/google-vertex-haystack) | [![Test / google-vertex](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/google_vertex.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/google_vertex.yml) |
| [gradient-haystack](integrations/gradient/) | Embedder, Generator | [![PyPI - Version](https://img.shields.io/pypi/v/gradient-haystack.svg)](https://pypi.org/project/gradient-haystack) | [![Test / gradient](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/gradient.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/gradient.yml) |
| [instructor-embedders-haystack](integrations/instructor_embedders/) | Embedder | [![PyPI - Version](https://img.shields.io/pypi/v/instructor-embedders-haystack.svg)](https://pypi.org/project/instructor-embedders-haystack) | [![Test / instructor-embedders](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/instructor_embedders.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/instructor_embedders.yml) |
| [jina-haystack](integrations/jina/) | Embedder | [![PyPI - Version](https://img.shields.io/pypi/v/jina-haystack.svg)](https://pypi.org/project/jina-haystack) | [![Test / jina](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/jina.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/jina.yml) |
| [jina-haystack](integrations/jina/) | Embedder, Ranker | [![PyPI - Version](https://img.shields.io/pypi/v/jina-haystack.svg)](https://pypi.org/project/jina-haystack) | [![Test / jina](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/jina.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/jina.yml) |
| [langfuse-haystack](integrations/langfuse/) | Tracer | [![PyPI - Version](https://img.shields.io/pypi/v/langfuse-haystack.svg?color=orange)](https://pypi.org/project/langfuse-haystack) | [![Test / langfuse](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/langfuse.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/langfuse.yml) |
| [llama-cpp-haystack](integrations/llama_cpp/) | Generator | [![PyPI - Version](https://img.shields.io/pypi/v/llama-cpp-haystack.svg?color=orange)](https://pypi.org/project/llama-cpp-haystack) | [![Test / llama-cpp](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/llama_cpp.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/llama_cpp.yml) |
| [mistral-haystack](integrations/mistral/) | Embedder, Generator | [![PyPI - Version](https://img.shields.io/pypi/v/mistral-haystack.svg)](https://pypi.org/project/mistral-haystack) | [![Test / mistral](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/mistral.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/mistral.yml) |
| [mongodb-atlas-haystack](integrations/mongodb_atlas/) | Document Store | [![PyPI - Version](https://img.shields.io/pypi/v/mongodb-atlas-haystack.svg?color=orange)](https://pypi.org/project/mongodb-atlas-haystack) | [![Test / mongodb-atlas](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/mongodb_atlas.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/mongodb_atlas.yml) |
Expand Down
2 changes: 1 addition & 1 deletion integrations/chroma/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ classifiers = [
]
dependencies = [
"haystack-ai",
"chromadb<0.4.20", # FIXME: investigate why filtering tests broke on 0.4.20
"chromadb",
"typing_extensions>=4.8.0"
]

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
GoogleVertexEmbeddingFunction,
HuggingFaceEmbeddingFunction,
InstructorEmbeddingFunction,
OllamaEmbeddingFunction,
ONNXMiniLM_L6_V2,
OpenAIEmbeddingFunction,
SentenceTransformerEmbeddingFunction,
Expand All @@ -25,6 +26,7 @@
"GoogleVertexEmbeddingFunction": GoogleVertexEmbeddingFunction,
"HuggingFaceEmbeddingFunction": HuggingFaceEmbeddingFunction,
"InstructorEmbeddingFunction": InstructorEmbeddingFunction,
"OllamaEmbeddingFunction": OllamaEmbeddingFunction,
"ONNXMiniLM_L6_V2": ONNXMiniLM_L6_V2,
"OpenAIEmbeddingFunction": OpenAIEmbeddingFunction,
"Text2VecEmbeddingFunction": Text2VecEmbeddingFunction,
Expand Down
4 changes: 4 additions & 0 deletions integrations/chroma/tests/test_document_store.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# SPDX-FileCopyrightText: 2023-present John Doe <[email protected]>
#
# SPDX-License-Identifier: Apache-2.0
import operator
import uuid
from typing import List
from unittest import mock
Expand Down Expand Up @@ -56,6 +57,9 @@ def assert_documents_are_equal(self, received: List[Document], expected: List[Do
This can happen for example when the Document Store sets a score to returned Documents.
Since we can't know what the score will be, we can't compare the Documents reliably.
"""
received.sort(key=operator.attrgetter("id"))
expected.sort(key=operator.attrgetter("id"))

for doc_received, doc_expected in zip(received, expected):
assert doc_received.content == doc_expected.content
assert doc_received.meta == doc_expected.meta
Expand Down
2 changes: 1 addition & 1 deletion integrations/cohere/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ classifiers = [
]
dependencies = [
"haystack-ai",
"cohere<5",
"cohere==5.*",
]

[project.urls]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from haystack.utils import Secret, deserialize_secrets_inplace
from haystack_integrations.components.embedders.cohere.utils import get_async_response, get_response

from cohere import COHERE_API_URL, AsyncClient, Client
from cohere import AsyncClient, Client


@component
Expand Down Expand Up @@ -39,7 +39,7 @@ def __init__(
api_key: Secret = Secret.from_env_var(["COHERE_API_KEY", "CO_API_KEY"]),
model: str = "embed-english-v2.0",
input_type: str = "search_document",
api_base_url: str = COHERE_API_URL,
api_base_url: str = "https://api.cohere.com",
truncate: str = "END",
use_async_client: bool = False,
max_retries: int = 3,
Expand Down Expand Up @@ -169,7 +169,7 @@ def run(self, documents: List[Document]):
if self.use_async_client:
cohere_client = AsyncClient(
api_key,
api_url=self.api_base_url,
base_url=self.api_base_url,
max_retries=self.max_retries,
timeout=self.timeout,
client_name="haystack",
Expand All @@ -180,7 +180,7 @@ def run(self, documents: List[Document]):
else:
cohere_client = Client(
api_key,
api_url=self.api_base_url,
base_url=self.api_base_url,
max_retries=self.max_retries,
timeout=self.timeout,
client_name="haystack",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from haystack.utils import Secret, deserialize_secrets_inplace
from haystack_integrations.components.embedders.cohere.utils import get_async_response, get_response

from cohere import COHERE_API_URL, AsyncClient, Client
from cohere import AsyncClient, Client


@component
Expand Down Expand Up @@ -36,7 +36,7 @@ def __init__(
api_key: Secret = Secret.from_env_var(["COHERE_API_KEY", "CO_API_KEY"]),
model: str = "embed-english-v2.0",
input_type: str = "search_query",
api_base_url: str = COHERE_API_URL,
api_base_url: str = "https://api.cohere.com",
truncate: str = "END",
use_async_client: bool = False,
max_retries: int = 3,
Expand Down Expand Up @@ -131,7 +131,7 @@ def run(self, text: str):
if self.use_async_client:
cohere_client = AsyncClient(
api_key,
api_url=self.api_base_url,
base_url=self.api_base_url,
max_retries=self.max_retries,
timeout=self.timeout,
client_name="haystack",
Expand All @@ -142,7 +142,7 @@ def run(self, text: str):
else:
cohere_client = Client(
api_key,
api_url=self.api_base_url,
base_url=self.api_base_url,
max_retries=self.max_retries,
timeout=self.timeout,
client_name="haystack",
Expand Down
Loading

0 comments on commit 17f6fe2

Please sign in to comment.