Skip to content

Commit

Permalink
docs: review google vertex integration (#535)
Browse files Browse the repository at this point in the history
* docs: review google vertex integration

* docs: fix line length issues

* docs: black format

* docs: add linked asset to repo
  • Loading branch information
wochinge authored Mar 5, 2024
1 parent ff41e80 commit cb28fee
Show file tree
Hide file tree
Showing 8 changed files with 373 additions and 35 deletions.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,44 @@

@component
class VertexAIImageCaptioner:
"""
`VertexAIImageCaptioner` enables text generation using Google Vertex AI imagetext generative model.
Authenticates using Google Cloud Application Default Credentials (ADCs).
For more information see the official [Google documentation](https://cloud.google.com/docs/authentication/provide-credentials-adc).
Usage example:
```python
import requests
from haystack.dataclasses.byte_stream import ByteStream
from haystack_integrations.components.generators.google_vertex import VertexAIImageCaptioner
captioner = VertexAIImageCaptioner(project_id=project_id)
image = ByteStream(
data=requests.get(
"https://raw.githubusercontent.com/deepset-ai/haystack-core-integrations/main/integrations/google_vertex/example_assets/robot1.jpg"
).content
)
result = captioner.run(image=image)
for caption in result["captions"]:
print(caption)
>>> two gold robots are standing next to each other in the desert
```
"""

def __init__(self, *, model: str = "imagetext", project_id: str, location: Optional[str] = None, **kwargs):
"""
Generate image captions using a Google Vertex AI model.
Authenticates using Google Cloud Application Default Credentials (ADCs).
For more information see the official Google documentation:
https://cloud.google.com/docs/authentication/provide-credentials-adc
For more information see the official [Google documentation](https://cloud.google.com/docs/authentication/provide-credentials-adc).
:param project_id: ID of the GCP project to use.
:param model: Name of the model to use, defaults to "imagetext".
:param model: Name of the model to use.
:param location: The default location to use when making API calls, if not set uses us-central-1.
Defaults to None.
:param kwargs: Additional keyword arguments to pass to the model.
Expand All @@ -39,15 +67,35 @@ def __init__(self, *, model: str = "imagetext", project_id: str, location: Optio
self._model = ImageTextModel.from_pretrained(self._model_name)

def to_dict(self) -> Dict[str, Any]:
"""
Serializes the component to a dictionary.
:returns:
Dictionary with serialized data.
"""
return default_to_dict(
self, model=self._model_name, project_id=self._project_id, location=self._location, **self._kwargs
)

@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "VertexAIImageCaptioner":
"""
Deserializes the component from a dictionary.
:param data:
Dictionary to deserialize from.
:returns:
Deserialized component.
"""
return default_from_dict(cls, data)

@component.output_types(captions=List[str])
def run(self, image: ByteStream):
"""Prompts the model to generate captions for the given image.
:param image: The image to generate captions for.
:returns: A dictionary with the following keys:
- `captions`: A list of captions generated by the model.
"""
captions = self._model.get_captions(image=Image(image.data), **self._kwargs)
return {"captions": captions}
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,30 @@

@component
class VertexAIGeminiChatGenerator:
"""
`VertexAIGeminiChatGenerator` enables chat completion using Google Gemini models.
`VertexAIGeminiChatGenerator` supports both `gemini-pro` and `gemini-pro-vision` models.
Prompting with images requires `gemini-pro-vision`. Function calling, instead, requires `gemini-pro`.
Authenticates using Google Cloud Application Default Credentials (ADCs).
For more information see the official [Google documentation](https://cloud.google.com/docs/authentication/provide-credentials-adc).
Usage example:
```python
from haystack.dataclasses import ChatMessage
from haystack_integrations.components.generators.google_vertex import VertexAIGeminiChatGenerator
gemini_chat = VertexAIGeminiChatGenerator(project_id=project_id)
messages = [ChatMessage.from_user("Tell me the name of a movie")]
res = gemini_chat.run(messages)
print(res["replies"][0].content)
>>> The Shawshank Redemption
```
"""

def __init__(
self,
*,
Expand All @@ -33,18 +57,25 @@ def __init__(
tools: Optional[List[Tool]] = None,
):
"""
Multi modal generator using Gemini model via Google Vertex AI.
`VertexAIGeminiChatGenerator` enables chat completion using Google Gemini models.
Authenticates using Google Cloud Application Default Credentials (ADCs).
For more information see the official Google documentation:
https://cloud.google.com/docs/authentication/provide-credentials-adc
For more information see the official [Google documentation](https://cloud.google.com/docs/authentication/provide-credentials-adc).
:param project_id: ID of the GCP project to use.
:param model: Name of the model to use, defaults to "gemini-pro-vision".
:param location: The default location to use when making API calls, if not set uses us-central-1.
Defaults to None.
:param kwargs: Additional keyword arguments to pass to the model.
For a list of supported arguments see the `GenerativeModel.generate_content()` documentation.
:param generation_config: Configuration for the generation process.
See the [GenerationConfig documentation](https://cloud.google.com/python/docs/reference/aiplatform/latest/vertexai.preview.generative_models.GenerationConfig
for a list of supported arguments.
:param safety_settings: Safety settings to use when generating content. See the documentation
for [HarmBlockThreshold](https://cloud.google.com/python/docs/reference/aiplatform/latest/vertexai.preview.generative_models.HarmBlockThreshold)
and [HarmCategory](https://cloud.google.com/python/docs/reference/aiplatform/latest/vertexai.preview.generative_models.HarmCategory)
for more details.
:param tools: List of tools to use when generating content. See the documentation for
[Tool](https://cloud.google.com/python/docs/reference/aiplatform/latest/vertexai.preview.generative_models.Tool)
the list of supported arguments.
"""

# Login to GCP. This will fail if user has not set up their gcloud SDK
Expand Down Expand Up @@ -84,6 +115,12 @@ def _generation_config_to_dict(self, config: Union[GenerationConfig, Dict[str, A
}

def to_dict(self) -> Dict[str, Any]:
"""
Serializes the component to a dictionary.
:returns:
Dictionary with serialized data.
"""
data = default_to_dict(
self,
model=self._model_name,
Expand All @@ -101,6 +138,14 @@ def to_dict(self) -> Dict[str, Any]:

@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "VertexAIGeminiChatGenerator":
"""
Deserializes the component from a dictionary.
:param data:
Dictionary to deserialize from.
:returns:
Deserialized component.
"""
if (tools := data["init_parameters"].get("tools")) is not None:
data["init_parameters"]["tools"] = [Tool.from_dict(t) for t in tools]
if (generation_config := data["init_parameters"].get("generation_config")) is not None:
Expand Down Expand Up @@ -151,6 +196,12 @@ def _message_to_content(self, message: ChatMessage) -> Content:

@component.output_types(replies=List[ChatMessage])
def run(self, messages: List[ChatMessage]):
"""Prompts Google Vertex AI Gemini model to generate a response to a list of messages.
:param messages: The last message is the prompt, the rest are the history.
:returns: A dictionary with the following keys:
- `replies`: A list of ChatMessage objects representing the model's replies.
"""
history = [self._message_to_content(m) for m in messages[:-1]]
session = self._model.start_chat(history=history)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,50 @@

@component
class VertexAICodeGenerator:
"""
This component enables code generation using Google Vertex AI generative model.
`VertexAICodeGenerator` supports `code-bison`, `code-bison-32k`, and `code-gecko`.
Usage example:
```python
from haystack_integrations.components.generators.google_vertex import VertexAICodeGenerator
generator = VertexAICodeGenerator(project_id=project_id)
result = generator.run(prefix="def to_json(data):")
for answer in result["answers"]:
print(answer)
>>> ```python
>>> import json
>>>
>>> def to_json(data):
>>> \"\"\"Converts a Python object to a JSON string.
>>>
>>> Args:
>>> data: The Python object to convert.
>>>
>>> Returns:
>>> A JSON string representing the Python object.
>>> \"\"\"
>>>
>>> return json.dumps(data)
>>> ```
```
"""

def __init__(self, *, model: str = "code-bison", project_id: str, location: Optional[str] = None, **kwargs):
"""
Generate code using a Google Vertex AI model.
Authenticates using Google Cloud Application Default Credentials (ADCs).
For more information see the official Google documentation:
https://cloud.google.com/docs/authentication/provide-credentials-adc
For more information see the official [Google documentation](https://cloud.google.com/docs/authentication/provide-credentials-adc).
:param project_id: ID of the GCP project to use.
:param model: Name of the model to use, defaults to "text-bison".
:param model: Name of the model to use.
:param location: The default location to use when making API calls, if not set uses us-central-1.
Defaults to None.
:param kwargs: Additional keyword arguments to pass to the model.
For a list of supported arguments see the `TextGenerationModel.predict()` documentation.
"""
Expand All @@ -38,16 +70,38 @@ def __init__(self, *, model: str = "code-bison", project_id: str, location: Opti
self._model = CodeGenerationModel.from_pretrained(self._model_name)

def to_dict(self) -> Dict[str, Any]:
"""
Serializes the component to a dictionary.
:returns:
Dictionary with serialized data.
"""
return default_to_dict(
self, model=self._model_name, project_id=self._project_id, location=self._location, **self._kwargs
)

@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "VertexAICodeGenerator":
"""
Deserializes the component from a dictionary.
:param data:
Dictionary to deserialize from.
:returns:
Deserialized component.
"""
return default_from_dict(cls, data)

@component.output_types(answers=List[str])
def run(self, prefix: str, suffix: Optional[str] = None):
"""
Generate code using a Google Vertex AI model.
:param prefix: Code before the current point.
:param suffix: Code after the current point.
:returns: A dictionary with the following keys:
- `answers`: A list of generated code snippets.
"""
res = self._model.predict(prefix=prefix, suffix=suffix, **self._kwargs)
# Handle the case where the model returns multiple candidates
answers = [c.text for c in res.candidates] if hasattr(res, "candidates") else [res.text]
Expand Down
Loading

0 comments on commit cb28fee

Please sign in to comment.