Skip to content

Commit

Permalink
Merge pull request #253 from randoentity/workaround-toolcall
Browse files Browse the repository at this point in the history
workaround for tool calling
  • Loading branch information
bdashore3 authored Nov 29, 2024
2 parents b81dcda + 2e06fb0 commit fe44e4a
Showing 1 changed file with 10 additions and 3 deletions.
13 changes: 10 additions & 3 deletions endpoints/OAI/utils/chat_completion.py
Original file line number Diff line number Diff line change
Expand Up @@ -454,16 +454,23 @@ async def generate_tool_calls(
if gen["stop_str"] in tool_data.tool_call_start:
if "text" in gen:
# non streaming, all generations will have the text they generated
pre_tool_prompt = await apply_chat_template(data, gen["text"])
pre_tool_prompt, mm_embeddings = await apply_chat_template(
data, gen["text"]
)
elif current_generations is not None:
# streaming, we wont have text in the generation,
# we'll have to use the current_generations
pre_tool_prompt = await apply_chat_template(data, current_generations)
pre_tool_prompt, mm_embeddings = await apply_chat_template(
data, current_generations
)

gen_tasks.append(
asyncio.create_task(
model.container.generate(
pre_tool_prompt, request.state.id, **gen_params
pre_tool_prompt,
request.state.id,
embeddings=mm_embeddings,
**gen_params,
)
)
)
Expand Down

0 comments on commit fe44e4a

Please sign in to comment.