Add handling of invalid JSON in tool call argument

OpenAI will occasionally respond with a tool call that contains invalid
JSON. Rather than trying to figure out how to fix their bad JSON, we now
just bypass adding it to the outstanding tool calls and instead respond
directly to the LLM with an error message indicating that the tool call
arguments were invalid JSON.

.tmuxinator.yml

@@ -0,0 +1,55 @@
+# /home/jwilger/.config/tmuxinator/apex.yml
+
+name: gpt_agent
+root: ~/projects/gpt_agent
+
+# Optional tmux socket
+# socket_name: foo
+
+# Note that the pre and post options have been deprecated and will be replaced by
+# project hooks.
+
+# Project hooks
+
+# Runs on project start, always
+# on_project_start: mix deps.get
+
+# Run on project start, the first time
+# on_project_first_start: mix setup
+
+# Run on project start, after the first time
+# on_project_restart: command
+
+# Run on project exit ( detaching from tmux session )
+# on_project_exit: command
+
+# Run on project stop
+# on_project_stop: command
+
+# Runs in each window and pane before window/pane specific commands. Useful for setting up interpreter versions.
+# pre_window: rbenv shell 2.0.0-p247
+
+# Pass command line options to tmux. Useful for specifying a different tmux.conf.
+# tmux_options: -f ~/.tmux.mac.conf
+
+# Change the command to call tmux. This can be used by derivatives/wrappers like byobu.
+# tmux_command: byobu
+
+# Specifies (by name or index) which window will be selected on project startup. If not set, the first window is used.
+# startup_window: editor
+
+# Specifies (by index) which pane of the specified window will be selected on project startup. If not set, the first pane is used.
+# startup_pane: 1
+
+# Controls whether the tmux session should be attached to automatically. Defaults to true.
+# attach: false
+
+windows:
+  - editor:
+      layout: 49d3,364x88,0,0{280x88,0,0,20,83x88,281,0,22}
+      # Synchronize all panes of this window, can be enabled before or after the pane commands run.
+      # 'before' represents legacy functionality and will be deprecated in a future release, in favour of 'after'
+      # synchronize: after
+      panes:
+        - vim
+        - mix test.interactive --stale

lib/gpt_agent.ex

@@ -380,19 +380,40 @@ defmodule GptAgent do
 
     tool_calls
     |> Enum.reduce(state, fn tool_call, state ->
-      tool_call =
-        ToolCallRequested.new!(
-          id: tool_call["id"],
-          thread_id: state.thread_id,
-          run_id: id,
-          name: tool_call["function"]["name"],
-          arguments: Jason.decode!(tool_call["function"]["arguments"])
-        )
+      case Jason.decode(tool_call["function"]["arguments"]) do
+        {:ok, arguments} ->
+          tool_call =
+            ToolCallRequested.new!(
+              id: tool_call["id"],
+              thread_id: state.thread_id,
+              run_id: id,
+              name: tool_call["function"]["name"],
+              arguments: arguments
+            )
 
-      state
-      |> Map.put(:tool_calls, [tool_call | state.tool_calls])
-      |> publish_event(tool_call)
+          state
+          |> Map.put(:tool_calls, [tool_call | state.tool_calls])
+          |> publish_event(tool_call)
+
+        {:error, %Jason.DecodeError{}} ->
+          log("Failed to decode tool call arguments: #{inspect(tool_call)}", :warning)
+
+          tool_output =
+            ToolCallOutputRecorded.new!(
+              id: tool_call["id"],
+              thread_id: state.thread_id,
+              run_id: id,
+              name: tool_call["function"]["name"],
+              output:
+                Jason.encode!(%{error: "Failed to decode arguments, invalid JSON in tool call."})
+            )
+
+          state
+          |> publish_event(tool_output)
+          |> Map.put(:tool_outputs, [tool_output | state.tool_outputs])
+      end
     end)
+    |> possibly_send_outputs_to_openai()
     |> noreply()
   end
 

mix.exs

@@ -4,7 +4,7 @@ defmodule GptAgent.MixProject do
   def project do
     [
       app: :gpt_agent,
-      version: "9.0.0",
+      version: "9.0.1",
       elixir: "~> 1.16",
       start_permanent: Mix.env() == :prod,
       aliases: aliases(),

test/gpt_agent_test.exs

@@ -992,6 +992,95 @@ defmodule GptAgentTest do
                      5_000
     end
 
+    @tag capture_log: true
+    test "when the run makes tool calls with invalid JSON in the arguments, it attempts to recover",
+         %{
+           bypass: bypass,
+           assistant_id: assistant_id,
+           thread_id: thread_id,
+           run_id: run_id
+         } do
+      {:ok, pid} =
+        GptAgent.connect(thread_id: thread_id, last_message_id: nil, assistant_id: assistant_id)
+
+      tool_1_id = UUID.uuid4()
+      tool_2_id = UUID.uuid4()
+
+      Bypass.expect(bypass, "GET", "/v1/threads/#{thread_id}/runs/#{run_id}", fn conn ->
+        conn
+        |> Plug.Conn.put_resp_content_type("application/json")
+        |> Plug.Conn.resp(
+          200,
+          Jason.encode!(%{
+            "id" => run_id,
+            "object" => "thread.run",
+            "created_at" => 1_699_075_072,
+            "assistant_id" => assistant_id,
+            "thread_id" => thread_id,
+            "status" => "requires_action",
+            "required_action" => %{
+              "type" => "submit_tool_outputs",
+              "submit_tool_outputs" => %{
+                "tool_calls" => [
+                  %{
+                    "id" => tool_1_id,
+                    "type" => "function",
+                    "function" => %{
+                      "name" => "tool_1",
+                      "arguments" => ~s({"foo":"bar\n\nba"z","baz":1})
+                    }
+                  },
+                  %{
+                    "id" => tool_2_id,
+                    "type" => "function",
+                    "function" => %{
+                      "name" => "tool_2",
+                      "arguments" => ~s({"ham":"spam","wham":2})
+                    }
+                  }
+                ]
+              }
+            },
+            "started_at" => 1_699_075_072,
+            "expires_at" => nil,
+            "cancelled_at" => nil,
+            "failed_at" => nil,
+            "completed_at" => 1_699_075_073,
+            "last_error" => nil,
+            "model" => "gpt-4-1106-preview",
+            "instructions" => nil,
+            "tools" => [],
+            "file_ids" => [],
+            "metadata" => %{}
+          })
+        )
+      end)
+
+      :ok = GptAgent.add_user_message(pid, "Hello")
+
+      expected_error_output =
+        Jason.encode!(%{error: "Failed to decode arguments, invalid JSON in tool call."})
+
+      assert_receive {^pid,
+                      %ToolCallOutputRecorded{
+                        id: ^tool_1_id,
+                        thread_id: ^thread_id,
+                        run_id: ^run_id,
+                        name: "tool_1",
+                        output: ^expected_error_output
+                      }}
+
+      assert_receive {^pid,
+                      %ToolCallRequested{
+                        id: ^tool_2_id,
+                        thread_id: ^thread_id,
+                        run_id: ^run_id,
+                        name: "tool_2",
+                        arguments: %{"ham" => "spam", "wham" => 2}
+                      }},
+                     5_000
+    end
+
     test "allow adding additional messages if the run is not complete", %{
       assistant_id: assistant_id,
       thread_id: thread_id,