Merge pull request #647 from ibuildthecloud/toolmiss

chore: return missing tool call to LLM, don't fail

pkg/engine/engine.go

@@ -45,8 +45,9 @@ type Return struct {
 }
 
 type Call struct {
-	ToolID string `json:"toolID,omitempty"`
-	Input  string `json:"input,omitempty"`
+	Missing bool   `json:"missing,omitempty"`
+	ToolID  string `json:"toolID,omitempty"`
+	Input   string `json:"input,omitempty"`
 }
 
 type CallResult struct {
@@ -216,10 +217,7 @@ func NewContext(ctx context.Context, prg *types.Program, input string) (Context,
 }
 
 func (c *Context) SubCallContext(ctx context.Context, input, toolID, callID string, toolCategory ToolCategory) (Context, error) {
-	tool, ok := c.Program.ToolSet[toolID]
-	if !ok {
-		return Context{}, fmt.Errorf("failed to file tool for id [%s]", toolID)
-	}
+	tool := c.Program.ToolSet[toolID]
 
 	if callID == "" {
 		callID = counter.Next()
@@ -387,19 +385,25 @@ func (e *Engine) complete(ctx context.Context, state *State) (*Return, error) {
 	state.Pending = map[string]types.CompletionToolCall{}
 	for _, content := range resp.Content {
 		if content.ToolCall != nil {
-			var toolID string
+			var (
+				toolID  string
+				missing bool
+			)
 			for _, tool := range state.Completion.Tools {
 				if strings.EqualFold(tool.Function.Name, content.ToolCall.Function.Name) {
 					toolID = tool.Function.ToolID
 				}
 			}
 			if toolID == "" {
-				return nil, fmt.Errorf("failed to find tool id for tool %s in tool_call result", content.ToolCall.Function.Name)
+				log.Debugf("failed to find tool id for tool %s in tool_call result", content.ToolCall.Function.Name)
+				toolID = content.ToolCall.Function.Name
+				missing = true
 			}
 			state.Pending[content.ToolCall.ID] = *content.ToolCall
 			ret.Calls[content.ToolCall.ID] = Call{
-				ToolID: toolID,
-				Input:  content.ToolCall.Function.Arguments,
+				ToolID:  toolID,
+				Missing: missing,
+				Input:   content.ToolCall.Function.Arguments,
 			}
 		} else {
 			cp := content.Text

pkg/runner/runner.go

@@ -802,6 +802,18 @@ func (r *Runner) subCalls(callCtx engine.Context, monitor Monitor, env []string,
 
 	for _, id := range ids {
 		call := state.Continuation.Calls[id]
+		if call.Missing {
+			resultLock.Lock()
+			callResults = append(callResults, SubCallResult{
+				ToolID: call.ToolID,
+				CallID: id,
+				State: &State{
+					Result: &[]string{fmt.Sprintf("ERROR: can not call unknown tool named [%s]", call.ToolID)}[0],
+				},
+			})
+			resultLock.Unlock()
+			continue
+		}
 		d.Run(func(ctx context.Context) error {
 			result, err := r.subCall(ctx, callCtx, monitor, env, call.ToolID, call.Input, id, toolCategory)
 			if err != nil {

pkg/tests/runner_test.go

@@ -948,3 +948,18 @@ func TestSysContext(t *testing.T) {
 	require.Len(t, context.Call.AgentGroup, 1)
 	assert.Equal(t, context.Call.AgentGroup[0].Named, "iAmSuperman")
 }
+
+func TestMissingTool(t *testing.T) {
+	r := tester.NewRunner(t)
+
+	r.RespondWith(tester.Result{
+		Func: types.CompletionFunctionCall{
+			Name: "not bob",
+		},
+	})
+
+	resp, err := r.Run("", "Input 1")
+	require.NoError(t, err)
+	r.AssertResponded(t)
+	autogold.Expect("TEST RESULT CALL: 2").Equal(t, resp)
+}

pkg/tests/testdata/TestMissingTool/call1-resp.golden

@@ -0,0 +1,14 @@
+`{
+  "role": "assistant",
+  "content": [
+    {
+      "toolCall": {
+        "id": "call_1",
+        "function": {
+          "name": "not bob"
+        }
+      }
+    }
+  ],
+  "usage": {}
+}`

pkg/tests/testdata/TestMissingTool/call1.golden

@@ -0,0 +1,32 @@
+`{
+  "model": "gpt-4o",
+  "tools": [
+    {
+      "function": {
+        "toolID": "testdata/TestMissingTool/test.gpt:Bob",
+        "name": "Bob",
+        "parameters": null
+      }
+    }
+  ],
+  "messages": [
+    {
+      "role": "system",
+      "content": [
+        {
+          "text": "Call tool Bob"
+        }
+      ],
+      "usage": {}
+    },
+    {
+      "role": "user",
+      "content": [
+        {
+          "text": "Input 1"
+        }
+      ],
+      "usage": {}
+    }
+  ]
+}`

pkg/tests/testdata/TestMissingTool/call2-resp.golden

@@ -0,0 +1,9 @@
+`{
+  "role": "assistant",
+  "content": [
+    {
+      "text": "TEST RESULT CALL: 2"
+    }
+  ],
+  "usage": {}
+}`

pkg/tests/testdata/TestMissingTool/call2.golden

@@ -0,0 +1,61 @@
+`{
+  "model": "gpt-4o",
+  "tools": [
+    {
+      "function": {
+        "toolID": "testdata/TestMissingTool/test.gpt:Bob",
+        "name": "Bob",
+        "parameters": null
+      }
+    }
+  ],
+  "messages": [
+    {
+      "role": "system",
+      "content": [
+        {
+          "text": "Call tool Bob"
+        }
+      ],
+      "usage": {}
+    },
+    {
+      "role": "user",
+      "content": [
+        {
+          "text": "Input 1"
+        }
+      ],
+      "usage": {}
+    },
+    {
+      "role": "assistant",
+      "content": [
+        {
+          "toolCall": {
+            "id": "call_1",
+            "function": {
+              "name": "not bob"
+            }
+          }
+        }
+      ],
+      "usage": {}
+    },
+    {
+      "role": "tool",
+      "content": [
+        {
+          "text": "ERROR: can not call unknown tool named [not bob]"
+        }
+      ],
+      "toolCall": {
+        "id": "call_1",
+        "function": {
+          "name": "not bob"
+        }
+      },
+      "usage": {}
+    }
+  ]
+}`

pkg/tests/testdata/TestMissingTool/test.gpt

@@ -0,0 +1,10 @@
+tools: Bob
+
+Call tool Bob
+
+---
+name: Bob
+
+#!sys.echo
+
+You called?

pkg/tests/tester/runner.go

@@ -104,7 +104,20 @@ func (c *Client) Call(_ context.Context, messageRequest types.CompletionRequest,
 	}
 
 	if result.Func.Name != "" {
-		c.t.Fatalf("failed to find tool %s", result.Func.Name)
+		return &types.CompletionMessage{
+			Role: types.CompletionMessageRoleTypeAssistant,
+			Content: []types.ContentPart{
+				{
+					ToolCall: &types.CompletionToolCall{
+						ID: fmt.Sprintf("call_%d", c.id),
+						Function: types.CompletionFunctionCall{
+							Name:      result.Func.Name,
+							Arguments: result.Func.Arguments,
+						},
+					},
+				},
+			},
+		}, nil
 	}
 
 	return &types.CompletionMessage{