diff --git a/docs/concepts/parallel.md b/docs/concepts/parallel.md index 4233cbd7b..506010234 100644 --- a/docs/concepts/parallel.md +++ b/docs/concepts/parallel.md @@ -5,58 +5,101 @@ description: Learn about OpenAI's experimental parallel function calling to redu # Parallel Tools -One of the latest capabilities that OpenAI has recently introduced is parallel function calling. -To learn more you can read up on [this](https://platform.openai.com/docs/guides/function-calling/parallel-function-calling) +Parallel Tool Calling is a feature that allows you to call multiple functions in a single request. This makes it faster to get a response from the language model, especially if your tool calls are independent of each other. !!! warning "Experimental Feature" - This feature is currently in preview and is subject to change. only supported by the `gpt-4-turbo-preview` model. + Parallel tool calling is only supported by Gemini and OpenAI at the moment ## Understanding Parallel Function Calling By using parallel function callings that allow you to call multiple functions in a single request, you can significantly reduce the latency of your application without having to use tricks with now one builds a schema. -```python hl_lines="19 31" -from __future__ import annotations +=== "OpenAI" -import openai -import instructor + ```python hl_lines="19 31" + from __future__ import annotations -from typing import Iterable, Literal -from pydantic import BaseModel + import openai + import instructor + from typing import Iterable, Literal + from pydantic import BaseModel -class Weather(BaseModel): - location: str - units: Literal["imperial", "metric"] + class Weather(BaseModel): + location: str + units: Literal["imperial", "metric"] -class GoogleSearch(BaseModel): - query: str + class GoogleSearch(BaseModel): + query: str -client = instructor.from_openai( - openai.OpenAI(), mode=instructor.Mode.PARALLEL_TOOLS -) # (1)! -function_calls = client.chat.completions.create( - model="gpt-4-turbo-preview", - messages=[ - {"role": "system", "content": "You must always use tools"}, - { - "role": "user", - "content": "What is the weather in toronto and dallas and who won the super bowl?", - }, - ], - response_model=Iterable[Weather | GoogleSearch], # (2)! -) + client = instructor.from_openai( + openai.OpenAI(), mode=instructor.Mode.PARALLEL_TOOLS + ) # (1)! -for fc in function_calls: - print(fc) - #> location='Toronto' units='metric' - #> location='Dallas' units='imperial' - #> query='who won the super bowl' -``` + function_calls = client.chat.completions.create( + model="gpt-4o-mini", + messages=[ + {"role": "system", "content": "You must always use tools"}, + { + "role": "user", + "content": "What is the weather in toronto and dallas and who won the super bowl?", + }, + ], + response_model=Iterable[Weather | GoogleSearch], # (2)! + ) + + for fc in function_calls: + print(fc) + #> location='Toronto' units='metric' + #> location='Dallas' units='imperial' + #> query='who won the super bowl' + ``` + +=== "Vertex AI" + + ```python + import instructor + import vertexai + from vertexai.generative_models import GenerativeModel + from typing import Iterable, Literal + from pydantic import BaseModel + + vertexai.init() + + class Weather(BaseModel): + location: str + units: Literal["imperial", "metric"] + + + class GoogleSearch(BaseModel): + query: str + + + client = instructor.from_vertexai( + GenerativeModel("gemini-1.5-pro-preview-0409"), + mode=instructor.Mode.VERTEXAI_PARALLEL_TOOLS + ) # (1)! + + function_calls = client.create( + messages=[ + { + "role": "user", + "content": "What is the weather in toronto and dallas and who won the super bowl?", + }, + ], + response_model=Iterable[Weather | GoogleSearch], # (2)! + ) + + for fc in function_calls: + print(fc) + #> location='Toronto' units='metric' + #> location='Dallas' units='imperial' + #> query='who won the super bowl' + ``` 1. Set the mode to `PARALLEL_TOOLS` to enable parallel function calling. 2. Set the response model to `Iterable[Weather | GoogleSearch]` to indicate that the response will be a list of `Weather` and `GoogleSearch` objects. This is necessary because the response will be a list of objects, and we need to specify the types of the objects in the list. diff --git a/instructor/dsl/parallel.py b/instructor/dsl/parallel.py index 5d207f5f8..2cd27d07b 100644 --- a/instructor/dsl/parallel.py +++ b/instructor/dsl/parallel.py @@ -54,7 +54,9 @@ def from_response( validation_context: Optional[Any] = None, strict: Optional[bool] = None, ) -> Generator[BaseModel, None, None]: - assert mode == Mode.VERTEXAI_PARALLEL_TOOLS, "Mode must be VERTEXAI_PARALLEL_TOOLS" + assert ( + mode == Mode.VERTEXAI_PARALLEL_TOOLS + ), "Mode must be VERTEXAI_PARALLEL_TOOLS" if not response or not response.candidates: return @@ -64,9 +66,7 @@ def from_response( continue for part in candidate.content.parts: - if (hasattr(part, 'function_call') and - part.function_call is not None): - + if hasattr(part, "function_call") and part.function_call is not None: name = part.function_call.name arguments = part.function_call.args @@ -116,6 +116,7 @@ def ParallelModel(typehint: type[Iterable[T]]) -> ParallelBase: the_types = get_types_array(typehint) return ParallelBase(*[model for model in the_types]) + def VertexAIParallelModel(typehint: type[Iterable[T]]) -> VertexAIParallelBase: the_types = get_types_array(typehint) return VertexAIParallelBase(*[model for model in the_types]) diff --git a/tests/llm/test_vertexai/test_modes.py b/tests/llm/test_vertexai/test_modes.py index 3bcc4f68a..e8a990c5b 100644 --- a/tests/llm/test_vertexai/test_modes.py +++ b/tests/llm/test_vertexai/test_modes.py @@ -13,7 +13,7 @@ class Item(BaseModel): class Order(BaseModel): - items: list[Item] = Field(..., default_factory=list) + items: list[Item] customer: str @@ -54,7 +54,7 @@ class Book(BaseModel): class LibraryRecord(BaseModel): - books: list[Book] = Field(..., default_factory=list) + books: list[Book] visitor: str library_id: str