From 51d17ef40f2a2bc99ff5cb3572c02bdc2b332862 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Fri, 9 Feb 2024 17:32:52 -0700 Subject: [PATCH] Fix daemon support and don't default to 1024 tokens --- pkg/cli/gptscript.go | 5 +++- pkg/engine/daemon.go | 62 ++++++++++++++++++++++++++++++++++++++------ pkg/engine/engine.go | 4 +-- pkg/engine/http.go | 56 +++++++++++++++++++++++++++++++++++---- pkg/loader/loader.go | 9 ++++--- pkg/openai/client.go | 5 ---- pkg/parser/parser.go | 22 +++++++++------- pkg/types/tool.go | 2 +- 8 files changed, 131 insertions(+), 34 deletions(-) diff --git a/pkg/cli/gptscript.go b/pkg/cli/gptscript.go index 2712f373..4c09d0fa 100644 --- a/pkg/cli/gptscript.go +++ b/pkg/cli/gptscript.go @@ -10,6 +10,7 @@ import ( "github.com/acorn-io/cmd" "github.com/gptscript-ai/gptscript/pkg/assemble" "github.com/gptscript-ai/gptscript/pkg/builtin" + "github.com/gptscript-ai/gptscript/pkg/engine" "github.com/gptscript-ai/gptscript/pkg/input" "github.com/gptscript-ai/gptscript/pkg/loader" "github.com/gptscript-ai/gptscript/pkg/monitor" @@ -98,6 +99,8 @@ func (r *GPTScript) Pre(cmd *cobra.Command, args []string) error { } func (r *GPTScript) Run(cmd *cobra.Command, args []string) error { + defer engine.CloseDaemons() + if r.ListModels { return r.listModels(cmd.Context()) } @@ -119,7 +122,7 @@ func (r *GPTScript) Run(cmd *cobra.Command, args []string) error { } if len(args) == 0 { - return fmt.Errorf("scripts argument required") + return cmd.Help() } prg, err := loader.Program(cmd.Context(), args[0], r.SubTool) diff --git a/pkg/engine/daemon.go b/pkg/engine/daemon.go index 884c02c6..a232c1cf 100644 --- a/pkg/engine/daemon.go +++ b/pkg/engine/daemon.go @@ -19,8 +19,23 @@ var ( startPort, endPort int64 nextPort int64 + daemonCtx context.Context + daemonClose func() + daemonWG sync.WaitGroup ) +func CloseDaemons() { + daemonLock.Lock() + if daemonCtx == nil { + daemonLock.Unlock() + return + } + daemonLock.Unlock() + + daemonClose() + daemonWG.Wait() +} + func (e *Engine) getNextPort() int64 { if startPort == 0 { startPort = 10240 @@ -32,24 +47,52 @@ func (e *Engine) getNextPort() int64 { return startPort + nextPort } +func getPath(instructions string) (string, string) { + instructions = strings.TrimSpace(instructions) + line := strings.TrimSpace(instructions) + + if !strings.HasPrefix(line, "(") { + return instructions, "" + } + + line, rest, ok := strings.Cut(line[1:], ")") + if !ok { + return instructions, "" + } + + path, value, ok := strings.Cut(strings.TrimSpace(line), "=") + if !ok || strings.TrimSpace(path) != "path" { + return instructions, "" + } + + return strings.TrimSpace(rest), strings.TrimSpace(value) +} + func (e *Engine) startDaemon(ctx context.Context, tool types.Tool) (string, error) { daemonLock.Lock() defer daemonLock.Unlock() + instructions := strings.TrimPrefix(tool.Instructions, types.DaemonPrefix) + instructions, path := getPath(instructions) + port, ok := daemonPorts[tool.ID] - url := fmt.Sprintf("http://127.0.0.1:%d", port) + url := fmt.Sprintf("http://127.0.0.1:%d%s", port, path) if ok { return url, nil } + if daemonCtx == nil { + daemonCtx, daemonClose = context.WithCancel(context.Background()) + } + + ctx = daemonCtx port = e.getNextPort() - url = fmt.Sprintf("http://127.0.0.1:%d", port) + url = fmt.Sprintf("http://127.0.0.1:%d%s", port, path) - instructions := types.CommandPrefix + strings.TrimPrefix(tool.Instructions, types.DaemonPrefix) cmd, close, err := e.newCommand(ctx, []string{ fmt.Sprintf("PORT=%d", port), }, - instructions, + types.CommandPrefix+instructions, "{}", ) if err != nil { @@ -58,7 +101,7 @@ func (e *Engine) startDaemon(ctx context.Context, tool types.Tool) (string, erro cmd.Stderr = os.Stderr cmd.Stdout = os.Stdout - log.Infof("launched [%s] port [%d] %v", tool.Name, port, cmd.Args) + log.Infof("launched [%s][%s] port [%d] %v", tool.Name, tool.ID, port, cmd.Args) if err := cmd.Start(); err != nil { close() return url, err @@ -67,6 +110,7 @@ func (e *Engine) startDaemon(ctx context.Context, tool types.Tool) (string, erro if daemonPorts == nil { daemonPorts = map[string]int64{} } + daemonPorts[tool.ID] = port killedCtx, cancel := context.WithCancelCause(ctx) defer cancel(nil) @@ -85,13 +129,15 @@ func (e *Engine) startDaemon(ctx context.Context, tool types.Tool) (string, erro delete(daemonPorts, tool.ID) }() + daemonWG.Add(1) context.AfterFunc(ctx, func() { if err := cmd.Process.Kill(); err != nil { log.Errorf("daemon failed to kill tool [%s] process: %v", tool.Name, err) } + daemonWG.Done() }) - for range 20 { + for i := 0; i < 20; i++ { resp, err := http.Get(url) if err == nil && resp.StatusCode == http.StatusOK { defer func() { @@ -112,7 +158,7 @@ func (e *Engine) startDaemon(ctx context.Context, tool types.Tool) (string, erro return url, fmt.Errorf("timeout waiting for 200 response from GET %s", url) } -func (e *Engine) runDaemon(ctx context.Context, tool types.Tool, input string) (cmdRet *Return, cmdErr error) { +func (e *Engine) runDaemon(ctx context.Context, prg *types.Program, tool types.Tool, input string) (cmdRet *Return, cmdErr error) { url, err := e.startDaemon(ctx, tool) if err != nil { return nil, err @@ -121,5 +167,5 @@ func (e *Engine) runDaemon(ctx context.Context, tool types.Tool, input string) ( tool.Instructions = strings.Join(append([]string{ types.CommandPrefix + url, }, strings.Split(tool.Instructions, "\n")[1:]...), "\n") - return e.runHTTP(ctx, tool, input) + return e.runHTTP(ctx, prg, tool, input) } diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index 4d10e299..5077cfa2 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -158,9 +158,9 @@ func (e *Engine) Start(ctx Context, input string) (*Return, error) { if tool.IsCommand() { if tool.IsHTTP() { - return e.runHTTP(ctx.Ctx, tool, input) + return e.runHTTP(ctx.Ctx, ctx.Program, tool, input) } else if tool.IsDaemon() { - return e.runDaemon(ctx.Ctx, tool, input) + return e.runDaemon(ctx.Ctx, ctx.Program, tool, input) } s, err := e.runCommand(ctx.Ctx, tool, input) if err != nil { diff --git a/pkg/engine/http.go b/pkg/engine/http.go index 08cab5e1..a1221b3a 100644 --- a/pkg/engine/http.go +++ b/pkg/engine/http.go @@ -2,24 +2,60 @@ package engine import ( "context" + "encoding/json" "fmt" "io" "net/http" + "net/url" "strings" "github.com/gptscript-ai/gptscript/pkg/types" ) -func (e *Engine) runHTTP(ctx context.Context, tool types.Tool, input string) (cmdRet *Return, cmdErr error) { - url := strings.Split(tool.Instructions, "\n")[0][2:] +const DaemonURLSuffix = ".daemon.gpt.local" - req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, strings.NewReader(input)) +func (e *Engine) runHTTP(ctx context.Context, prg *types.Program, tool types.Tool, input string) (cmdRet *Return, cmdErr error) { + toolURL := strings.Split(tool.Instructions, "\n")[0][2:] + + parsed, err := url.Parse(toolURL) + if err != nil { + return nil, err + } + + if strings.HasSuffix(parsed.Hostname(), DaemonURLSuffix) { + referencedToolName := strings.TrimSuffix(parsed.Hostname(), DaemonURLSuffix) + referencedToolID, ok := tool.ToolMapping[referencedToolName] + if !ok { + return nil, fmt.Errorf("invalid reference [%s] to tool [%s] from [%s], missing \"tools: %s\" parameter", toolURL, referencedToolName, tool.Source, referencedToolName) + } + referencedTool, ok := prg.ToolSet[referencedToolID] + if !ok { + return nil, fmt.Errorf("failed to find tool [%s] for [%s]", referencedToolName, parsed.Hostname()) + } + toolURL, err = e.startDaemon(ctx, referencedTool) + if err != nil { + return nil, err + } + toolURLParsed, err := url.Parse(toolURL) + if err != nil { + return nil, err + } + parsed.Host = toolURLParsed.Host + toolURL = parsed.String() + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, toolURL, strings.NewReader(input)) if err != nil { return nil, err } req.Header.Set("X-GPTScript-Tool-Name", tool.Name) - req.Header.Set("Content-Type", "text/plain") + + if err := json.Unmarshal([]byte(input), &map[string]any{}); err == nil { + req.Header.Set("Content-Type", "application/json") + } else { + req.Header.Set("Content-Type", "text/plain") + } resp, err := http.DefaultClient.Do(req) if err != nil { @@ -29,7 +65,7 @@ func (e *Engine) runHTTP(ctx context.Context, tool types.Tool, input string) (cm if resp.StatusCode > 299 { _, _ = io.ReadAll(resp.Body) - return nil, fmt.Errorf("error in request to [%s] [%d]: %s", url, resp.StatusCode, resp.Status) + return nil, fmt.Errorf("error in request to [%s] [%d]: %s", toolURL, resp.StatusCode, resp.Status) } content, err := io.ReadAll(resp.Body) @@ -37,6 +73,16 @@ func (e *Engine) runHTTP(ctx context.Context, tool types.Tool, input string) (cm return nil, err } + if resp.Header.Get("Content-Type") == "application/json" && strings.HasPrefix(string(content), "\"") { + // This is dumb hack when something returns a string in JSON format, just decode it to a string + var s string + if err := json.Unmarshal(content, &s); err == nil { + return &Return{ + Result: &s, + }, nil + } + } + s := string(content) return &Return{ Result: &s, diff --git a/pkg/loader/loader.go b/pkg/loader/loader.go index c3eebad6..31b8ac9a 100644 --- a/pkg/loader/loader.go +++ b/pkg/loader/loader.go @@ -210,7 +210,7 @@ func readTool(ctx context.Context, prg *types.Program, base *source, targetToolN } if i != 0 && tool.Name == "" { - return types.Tool{}, parser.NewErrLine(tool.Source.LineNo, fmt.Errorf("only the first tool in a file can have no name")) + return types.Tool{}, parser.NewErrLine(tool.Source.File, tool.Source.LineNo, fmt.Errorf("only the first tool in a file can have no name")) } if targetToolName != "" && tool.Name == targetToolName { @@ -218,7 +218,7 @@ func readTool(ctx context.Context, prg *types.Program, base *source, targetToolN } if existing, ok := localTools[tool.Name]; ok { - return types.Tool{}, parser.NewErrLine(tool.Source.LineNo, + return types.Tool{}, parser.NewErrLine(tool.Source.File, tool.Source.LineNo, fmt.Errorf("duplicate tool name [%s] in %s found at lines %d and %d", tool.Name, tool.Source.File, tool.Source.LineNo, existing.Source.LineNo)) } @@ -313,10 +313,13 @@ func link(ctx context.Context, prg *types.Program, base *source, tool types.Tool continue } - toolName, subTool, ok := strings.Cut(targetToolName, " from ") + subTool, toolName, ok := strings.Cut(targetToolName, " from ") if ok { toolName = strings.TrimSpace(toolName) subTool = strings.TrimSpace(subTool) + } else { + toolName = targetToolName + subTool = "" } resolvedTool, err := resolve(ctx, prg, base, toolName, subTool) diff --git a/pkg/openai/client.go b/pkg/openai/client.go index 1e2d327e..fddb4500 100644 --- a/pkg/openai/client.go +++ b/pkg/openai/client.go @@ -23,7 +23,6 @@ import ( const ( DefaultVisionModel = openai.GPT4VisionPreview DefaultModel = openai.GPT4TurboPreview - DefaultMaxTokens = 1024 DefaultPromptParameter = "defaultPromptParameter" ) @@ -274,10 +273,6 @@ func (c *Client) Call(ctx context.Context, messageRequest types.CompletionReques } } - if request.MaxTokens == 0 { - request.MaxTokens = DefaultMaxTokens - } - if !messageRequest.Vision { for _, tool := range messageRequest.Tools { params := tool.Function.Parameters diff --git a/pkg/parser/parser.go b/pkg/parser/parser.go index 71331918..d8e3877a 100644 --- a/pkg/parser/parser.go +++ b/pkg/parser/parser.go @@ -123,6 +123,7 @@ func isParam(line string, tool *types.Tool) (_ bool, err error) { } type ErrLine struct { + Path string Line int Err error } @@ -132,11 +133,15 @@ func (e *ErrLine) Unwrap() error { } func (e *ErrLine) Error() string { - return fmt.Sprintf("line %d: %v", e.Line, e.Err) + if e.Path == "" { + return fmt.Sprintf("line %d: %v", e.Line, e.Err) + } + return fmt.Sprintf("line %s:%d: %v", e.Path, e.Line, e.Err) } -func NewErrLine(lineNo int, err error) error { +func NewErrLine(path string, lineNo int, err error) error { return &ErrLine{ + Path: path, Line: lineNo, Err: err, } @@ -161,7 +166,7 @@ func commentEmbedded(line string) (string, bool) { prefix := i + "gptscript:" cut, ok := strings.CutPrefix(line, prefix) if ok { - return cut, ok + return strings.TrimSpace(cut) + "\n", ok } } return line, false @@ -183,6 +188,10 @@ func Parse(input io.Reader) ([]types.Tool, error) { } line := scan.Text() + "\n" + if embeddedLine, ok := commentEmbedded(line); ok { + // Strip special comments to allow embedding the preamble in python or other interpreted languages + line = embeddedLine + } if line == "---\n" { context.finish(&tools) @@ -190,11 +199,6 @@ func Parse(input io.Reader) ([]types.Tool, error) { } if !context.inBody { - // Strip special comments to allow embedding the preamble in python or other interpreted languages - if newLine, ok := commentEmbedded(line); ok { - line = newLine - } - // If the very first line is #! just skip because this is a unix interpreter declaration if strings.HasPrefix(line, "#!") && lineNo == 1 { continue @@ -212,7 +216,7 @@ func Parse(input io.Reader) ([]types.Tool, error) { // Look for params if isParam, err := isParam(line, &context.tool); err != nil { - return nil, NewErrLine(lineNo, err) + return nil, NewErrLine("", lineNo, err) } else if isParam { continue } diff --git a/pkg/types/tool.go b/pkg/types/tool.go index 78b9907d..e54a1865 100644 --- a/pkg/types/tool.go +++ b/pkg/types/tool.go @@ -8,7 +8,7 @@ import ( ) const ( - DaemonPrefix = "#!sys.daemon " + DaemonPrefix = "#!sys.daemon" CommandPrefix = "#!" )