Skip to content

Commit

Permalink
docs: review google vertex integration
Browse files Browse the repository at this point in the history
  • Loading branch information
wochinge committed Mar 5, 2024
1 parent 52a8354 commit f52d9eb
Show file tree
Hide file tree
Showing 7 changed files with 363 additions and 35 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,40 @@

@component
class VertexAIImageCaptioner:
"""
`VertexAIImageCaptioner` enables text generation using Google Vertex AI imagetext generative model.
Authenticates using Google Cloud Application Default Credentials (ADCs).
For more information see the official [Google documentation](https://cloud.google.com/docs/authentication/provide-credentials-adc).
Usage example:
```python
import requests
from haystack.dataclasses.byte_stream import ByteStream
from haystack_integrations.components.generators.google_vertex import VertexAIImageCaptioner
captioner = VertexAIImageCaptioner(project_id=project_id)
image = ByteStream(data=requests.get("https://raw.githubusercontent.com/silvanocerza/robots/main/robot1.jpg").content)
result = captioner.run(image=image)
for caption in result["captions"]:
print(caption)
>>> two gold robots are standing next to each other in the desert
```
"""

def __init__(self, *, model: str = "imagetext", project_id: str, location: Optional[str] = None, **kwargs):
"""
Generate image captions using a Google Vertex AI model.
Authenticates using Google Cloud Application Default Credentials (ADCs).
For more information see the official Google documentation:
https://cloud.google.com/docs/authentication/provide-credentials-adc
For more information see the official [Google documentation](https://cloud.google.com/docs/authentication/provide-credentials-adc).
:param project_id: ID of the GCP project to use.
:param model: Name of the model to use, defaults to "imagetext".
:param model: Name of the model to use.
:param location: The default location to use when making API calls, if not set uses us-central-1.
Defaults to None.
:param kwargs: Additional keyword arguments to pass to the model.
Expand All @@ -39,15 +63,35 @@ def __init__(self, *, model: str = "imagetext", project_id: str, location: Optio
self._model = ImageTextModel.from_pretrained(self._model_name)

def to_dict(self) -> Dict[str, Any]:
"""
Serializes the component to a dictionary.
:returns:
Dictionary with serialized data.
"""
return default_to_dict(
self, model=self._model_name, project_id=self._project_id, location=self._location, **self._kwargs
)

@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "VertexAIImageCaptioner":
"""
Deserializes the component from a dictionary.
:param data:
Dictionary to deserialize from.
:returns:
Deserialized component.
"""
return default_from_dict(cls, data)

@component.output_types(captions=List[str])
def run(self, image: ByteStream):
"""Prompts the model to generate captions for the given image.
:param image: The image to generate captions for.
:returns: A dictionary with the following keys:
- `captions`: A list of captions generated by the model.
"""
captions = self._model.get_captions(image=Image(image.data), **self._kwargs)
return {"captions": captions}
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,30 @@

@component
class VertexAIGeminiChatGenerator:
"""
`VertexAIGeminiChatGenerator` enables chat completion using Google Gemini models.
`VertexAIGeminiChatGenerator` supports both `gemini-pro` and `gemini-pro-vision` models.
Prompting with images requires `gemini-pro-vision`. Function calling, instead, requires `gemini-pro`.
Authenticates using Google Cloud Application Default Credentials (ADCs).
For more information see the official [Google documentation](https://cloud.google.com/docs/authentication/provide-credentials-adc).
Usage example:
```python
from haystack.dataclasses import ChatMessage
from haystack_integrations.components.generators.google_vertex import VertexAIGeminiChatGenerator
gemini_chat = VertexAIGeminiChatGenerator(project_id=project_id)
messages = [ChatMessage.from_user("Tell me the name of a movie")]
res = gemini_chat.run(messages)
print(res["replies"][0].content)
>>> The Shawshank Redemption
```
"""

def __init__(
self,
*,
Expand All @@ -33,18 +57,25 @@ def __init__(
tools: Optional[List[Tool]] = None,
):
"""
Multi modal generator using Gemini model via Google Vertex AI.
`VertexAIGeminiChatGenerator` enables chat completion using Google Gemini models.
Authenticates using Google Cloud Application Default Credentials (ADCs).
For more information see the official Google documentation:
https://cloud.google.com/docs/authentication/provide-credentials-adc
For more information see the official [Google documentation](https://cloud.google.com/docs/authentication/provide-credentials-adc).
:param project_id: ID of the GCP project to use.
:param model: Name of the model to use, defaults to "gemini-pro-vision".
:param location: The default location to use when making API calls, if not set uses us-central-1.
Defaults to None.
:param kwargs: Additional keyword arguments to pass to the model.
For a list of supported arguments see the `GenerativeModel.generate_content()` documentation.
:param generation_config: Configuration for the generation process.
See the [GenerationConfig documentation](https://cloud.google.com/python/docs/reference/aiplatform/latest/vertexai.preview.generative_models.GenerationConfig
for a list of supported arguments.
:param safety_settings: Safety settings to use when generating content. See the documentation
for [HarmBlockThreshold](https://cloud.google.com/python/docs/reference/aiplatform/latest/vertexai.preview.generative_models.HarmBlockThreshold)
and [HarmCategory](https://cloud.google.com/python/docs/reference/aiplatform/latest/vertexai.preview.generative_models.HarmCategory)
for more details.
:param tools: List of tools to use when generating content. See the documentation for
[Tool](https://cloud.google.com/python/docs/reference/aiplatform/latest/vertexai.preview.generative_models.Tool)
the list of supported arguments.
"""

# Login to GCP. This will fail if user has not set up their gcloud SDK
Expand Down Expand Up @@ -84,6 +115,12 @@ def _generation_config_to_dict(self, config: Union[GenerationConfig, Dict[str, A
}

def to_dict(self) -> Dict[str, Any]:
"""
Serializes the component to a dictionary.
:returns:
Dictionary with serialized data.
"""
data = default_to_dict(
self,
model=self._model_name,
Expand All @@ -101,6 +138,14 @@ def to_dict(self) -> Dict[str, Any]:

@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "VertexAIGeminiChatGenerator":
"""
Deserializes the component from a dictionary.
:param data:
Dictionary to deserialize from.
:returns:
Deserialized component.
"""
if (tools := data["init_parameters"].get("tools")) is not None:
data["init_parameters"]["tools"] = [Tool.from_dict(t) for t in tools]
if (generation_config := data["init_parameters"].get("generation_config")) is not None:
Expand Down Expand Up @@ -151,6 +196,12 @@ def _message_to_content(self, message: ChatMessage) -> Content:

@component.output_types(replies=List[ChatMessage])
def run(self, messages: List[ChatMessage]):
"""Prompts Google Vertex AI Gemini model to generate a response to a list of messages.
:param messages: The last message is the prompt, the rest are the history.
:returns: A dictionary with the following keys:
- `replies`: A list of ChatMessage objects representing the model's replies.
"""
history = [self._message_to_content(m) for m in messages[:-1]]
session = self._model.start_chat(history=history)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,50 @@

@component
class VertexAICodeGenerator:
"""
This component enables code generation using Google Vertex AI generative model.
`VertexAICodeGenerator` supports `code-bison`, `code-bison-32k`, and `code-gecko`.
Usage example:
```python
from haystack_integrations.components.generators.google_vertex import VertexAICodeGenerator
generator = VertexAICodeGenerator(project_id=project_id)
result = generator.run(prefix="def to_json(data):")
for answer in result["answers"]:
print(answer)
>>> ```python
>>> import json
>>>
>>> def to_json(data):
>>> \"\"\"Converts a Python object to a JSON string.
>>>
>>> Args:
>>> data: The Python object to convert.
>>>
>>> Returns:
>>> A JSON string representing the Python object.
>>> \"\"\"
>>>
>>> return json.dumps(data)
>>> ```
```
"""

def __init__(self, *, model: str = "code-bison", project_id: str, location: Optional[str] = None, **kwargs):
"""
Generate code using a Google Vertex AI model.
Authenticates using Google Cloud Application Default Credentials (ADCs).
For more information see the official Google documentation:
https://cloud.google.com/docs/authentication/provide-credentials-adc
For more information see the official [Google documentation](https://cloud.google.com/docs/authentication/provide-credentials-adc).
:param project_id: ID of the GCP project to use.
:param model: Name of the model to use, defaults to "text-bison".
:param model: Name of the model to use.
:param location: The default location to use when making API calls, if not set uses us-central-1.
Defaults to None.
:param kwargs: Additional keyword arguments to pass to the model.
For a list of supported arguments see the `TextGenerationModel.predict()` documentation.
"""
Expand All @@ -38,16 +70,38 @@ def __init__(self, *, model: str = "code-bison", project_id: str, location: Opti
self._model = CodeGenerationModel.from_pretrained(self._model_name)

def to_dict(self) -> Dict[str, Any]:
"""
Serializes the component to a dictionary.
:returns:
Dictionary with serialized data.
"""
return default_to_dict(
self, model=self._model_name, project_id=self._project_id, location=self._location, **self._kwargs
)

@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "VertexAICodeGenerator":
"""
Deserializes the component from a dictionary.
:param data:
Dictionary to deserialize from.
:returns:
Deserialized component.
"""
return default_from_dict(cls, data)

@component.output_types(answers=List[str])
def run(self, prefix: str, suffix: Optional[str] = None):
"""
Generate code using a Google Vertex AI model.
:param prefix: Code before the current point.
:param suffix: Code after the current point.
:returns: A dictionary with the following keys:
- `answers`: A list of generated code snippets.
"""
res = self._model.predict(prefix=prefix, suffix=suffix, **self._kwargs)
# Handle the case where the model returns multiple candidates
answers = [c.text for c in res.candidates] if hasattr(res, "candidates") else [res.text]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,34 @@

@component
class VertexAIGeminiGenerator:
"""
`VertexAIGeminiGenerator` enables text generation using Google Gemini models.
`VertexAIGeminiGenerator` supports both `gemini-pro` and `gemini-pro-vision` models.
Prompting with images requires `gemini-pro-vision`. Function calling, instead, requires `gemini-pro`.
Usage example:
```python
from haystack_integrations.components.generators.google_vertex import VertexAIGeminiGenerator
gemini = VertexAIGeminiGenerator(project_id=project_id)
result = gemini.run(parts = ["What is the most interesting thing you know?"])
for answer in result["answers"]:
print(answer)
>>> 1. **The Origin of Life:** How and where did life begin? The answers to this question are still shrouded in mystery, but scientists continuously uncover new insights into the remarkable story of our planet's earliest forms of life.
>>> 2. **The Unseen Universe:** The vast majority of the universe is comprised of matter and energy that we cannot directly observe. Dark matter and dark energy make up over 95% of the universe, yet we still don't fully understand their properties or how they influence the cosmos.
>>> 3. **Quantum Entanglement:** This eerie phenomenon in quantum mechanics allows two particles to become so intertwined that they share the same fate, regardless of how far apart they are. This has mind-bending implications for our understanding of reality and could potentially lead to advancements in communication and computing.
>>> 4. **Time Dilation:** Einstein's theory of relativity revealed that time can pass at different rates for different observers. Astronauts traveling at high speeds, for example, experience time dilation relative to people on Earth. This phenomenon could have significant implications for future space travel.
>>> 5. **The Fermi Paradox:** Despite the vastness of the universe and the abundance of potential life-supporting planets, we have yet to find any concrete evidence of extraterrestrial life. This contradiction between scientific expectations and observational reality is known as the Fermi Paradox and remains one of the most intriguing mysteries in modern science.
>>> 6. **Biological Evolution:** The idea that life evolves over time through natural selection is one of the most profound and transformative scientific discoveries. It explains the diversity of life on Earth and provides insights into our own origins and the interconnectedness of all living things.
>>> 7. **Neuroplasticity:** The brain's ability to adapt and change throughout life, known as neuroplasticity, is a remarkable phenomenon that has important implications for learning, memory, and recovery from brain injuries.
>>> 8. **The Goldilocks Zone:** The concept of the habitable zone, or the Goldilocks zone, refers to the range of distances from a star within which liquid water can exist on a planet's surface. This zone is critical for the potential existence of life as we know it and has been used to guide the search for exoplanets that could support life.
>>> 9. **String Theory:** This theoretical framework in physics aims to unify all the fundamental forces of nature into a single coherent theory. It suggests that the universe has extra dimensions beyond the familiar three spatial dimensions and time.
>>> 10. **Consciousness:** The nature of human consciousness and how it arises from the brain's physical processes remain one of the most profound and elusive mysteries in science. Understanding consciousness is crucial for unraveling the complexities of the human mind and our place in the universe.
```
"""
def __init__(
self,
*,
Expand All @@ -33,29 +61,31 @@ def __init__(
tools: Optional[List[Tool]] = None,
):
"""
Multi modal generator using Gemini model via Google Vertex AI.
Multi-modal generator using Gemini model via Google Vertex AI.
Authenticates using Google Cloud Application Default Credentials (ADCs).
For more information see the official Google documentation:
https://cloud.google.com/docs/authentication/provide-credentials-adc
For more information see the official [Google documentation](https://cloud.google.com/docs/authentication/provide-credentials-adc).
:param project_id: ID of the GCP project to use.
:param model: Name of the model to use, defaults to "gemini-pro-vision".
:param model: Name of the model to use.
:param location: The default location to use when making API calls, if not set uses us-central-1.
Defaults to None.
:param generation_config: The generation config to use, defaults to None.
Can either be a GenerationConfig object or a dictionary of parameters.
:param generation_config: The generation config to use.
Can either be a [`GenerationConfig`](https://cloud.google.com/python/docs/reference/aiplatform/latest/vertexai.preview.generative_models.GenerationConfig)
object or a dictionary of parameters.
Accepted fields are:
- temperature
- top_p
- top_k
- candidate_count
- max_output_tokens
- stop_sequences
:param safety_settings: The safety settings to use, defaults to None.
A dictionary of HarmCategory to HarmBlockThreshold.
:param tools: The tools to use, defaults to None.
A list of Tool objects that can be used to modify the generation process.
:param safety_settings: The safety settings to use. See the documentation
for [HarmBlockThreshold](https://cloud.google.com/python/docs/reference/aiplatform/latest/vertexai.preview.generative_models.HarmBlockThreshold)
and [HarmCategory](https://cloud.google.com/python/docs/reference/aiplatform/latest/vertexai.preview.generative_models.HarmCategory)
for more details.
:param tools: List of tools to use when generating content. See the documentation for
[Tool](https://cloud.google.com/python/docs/reference/aiplatform/latest/vertexai.preview.generative_models.Tool)
the list of supported arguments.
"""

# Login to GCP. This will fail if user has not set up their gcloud SDK
Expand Down Expand Up @@ -95,6 +125,12 @@ def _generation_config_to_dict(self, config: Union[GenerationConfig, Dict[str, A
}

def to_dict(self) -> Dict[str, Any]:
"""
Serializes the component to a dictionary.
:returns:
Dictionary with serialized data.
"""
data = default_to_dict(
self,
model=self._model_name,
Expand All @@ -112,6 +148,14 @@ def to_dict(self) -> Dict[str, Any]:

@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "VertexAIGeminiGenerator":
"""
Deserializes the component from a dictionary.
:param data:
Dictionary to deserialize from.
:returns:
Deserialized component.
"""
if (tools := data["init_parameters"].get("tools")) is not None:
data["init_parameters"]["tools"] = [Tool.from_dict(t) for t in tools]
if (generation_config := data["init_parameters"].get("generation_config")) is not None:
Expand All @@ -132,6 +176,13 @@ def _convert_part(self, part: Union[str, ByteStream, Part]) -> Part:

@component.output_types(answers=List[Union[str, Dict[str, str]]])
def run(self, parts: Variadic[Union[str, ByteStream, Part]]):
"""
Generates content using the Gemini model.
:param parts: Prompt for the model.
:returns: A dictionary with the following keys:
- `answers`: A list of generated content.
"""
converted_parts = [self._convert_part(p) for p in parts]

contents = [Content(parts=converted_parts, role="user")]
Expand Down
Loading

0 comments on commit f52d9eb

Please sign in to comment.