Skip to content

Commit

Permalink
Merge pull request #759 from ibuildthecloud/local-dev
Browse files Browse the repository at this point in the history
feat: manage node/python runtime for local files in dev
  • Loading branch information
ibuildthecloud authored Aug 10, 2024
2 parents 3bc07ac + cb46358 commit c2c5d99
Show file tree
Hide file tree
Showing 20 changed files with 589 additions and 258 deletions.
145 changes: 36 additions & 109 deletions pkg/engine/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (
"os"
"os/exec"
"path"
"path/filepath"
"runtime"
"sort"
"strings"
Expand Down Expand Up @@ -121,7 +120,7 @@ func (e *Engine) runCommand(ctx Context, tool types.Tool, input string, toolCate
var extraEnv = []string{
strings.TrimSpace("GPTSCRIPT_CONTEXT=" + strings.Join(instructions, "\n")),
}
cmd, stop, err := e.newCommand(ctx.Ctx, extraEnv, tool, input)
cmd, stop, err := e.newCommand(ctx.Ctx, extraEnv, tool, input, true)
if err != nil {
if toolCategory == NoCategory {
return fmt.Sprintf("ERROR: got (%v) while parsing command", err), nil
Expand Down Expand Up @@ -244,7 +243,11 @@ func appendInputAsEnv(env []string, input string) []string {
return env
}

func (e *Engine) newCommand(ctx context.Context, extraEnv []string, tool types.Tool, input string) (*exec.Cmd, func(), error) {
func (e *Engine) newCommand(ctx context.Context, extraEnv []string, tool types.Tool, input string, useShell bool) (*exec.Cmd, func(), error) {
if runtime.GOOS == "windows" {
useShell = false
}

envvars := append(e.Env[:], extraEnv...)
envvars = appendInputAsEnv(envvars, input)
if log.IsDebug() {
Expand All @@ -254,9 +257,17 @@ func (e *Engine) newCommand(ctx context.Context, extraEnv []string, tool types.T
interpreter, rest, _ := strings.Cut(tool.Instructions, "\n")
interpreter = strings.TrimSpace(interpreter)[2:]

args, err := shlex.Split(interpreter)
if err != nil {
return nil, nil, err
var (
args []string
err error
)
if useShell {
args = strings.Fields(interpreter)
} else {
args, err = shlex.Split(interpreter)
if err != nil {
return nil, nil, err
}
}

envvars, err = e.getRuntimeEnv(ctx, tool, args, envvars)
Expand All @@ -265,17 +276,6 @@ func (e *Engine) newCommand(ctx context.Context, extraEnv []string, tool types.T
}

envvars, envMap := envAsMapAndDeDup(envvars)
for i, arg := range args {
args[i] = os.Expand(arg, func(s string) string {
return envMap[s]
})
}

// After we determined the interpreter we again interpret the args by env vars
args, err = replaceVariablesForInterpreter(interpreter, envMap)
if err != nil {
return nil, nil, err
}

if runtime.GOOS == "windows" && (args[0] == "/bin/bash" || args[0] == "/bin/sh") {
args[0] = path.Base(args[0])
Expand All @@ -286,8 +286,7 @@ func (e *Engine) newCommand(ctx context.Context, extraEnv []string, tool types.T
}

var (
cmdArgs = args[1:]
stop = func() {}
stop = func() {}
)

if strings.TrimSpace(rest) != "" {
Expand All @@ -305,105 +304,33 @@ func (e *Engine) newCommand(ctx context.Context, extraEnv []string, tool types.T
stop()
return nil, nil, err
}
cmdArgs = append(cmdArgs, f.Name())
}

// This is a workaround for Windows, where the command interpreter is constructed with unix style paths
// It converts unix style paths to windows style paths
if runtime.GOOS == "windows" {
parts := strings.Split(args[0], "/")
if parts[len(parts)-1] == "gptscript-go-tool" {
parts[len(parts)-1] = "gptscript-go-tool.exe"
}

args[0] = filepath.Join(parts...)
args = append(args, f.Name())
}

cmd := exec.CommandContext(ctx, env.Lookup(envvars, args[0]), cmdArgs...)
cmd.Env = compressEnv(envvars)
return cmd, stop, nil
}

func replaceVariablesForInterpreter(interpreter string, envMap map[string]string) ([]string, error) {
var parts []string
for i, part := range splitByQuotes(interpreter) {
if i%2 == 0 {
part = os.Expand(part, func(s string) string {
// Expand and/or normalize env references
for i, arg := range args {
args[i] = os.Expand(arg, func(s string) string {
if strings.HasPrefix(s, "!") {
return envMap[s[1:]]
}
if !useShell {
return envMap[s]
})
// We protect newly resolved env vars from getting replaced when we do the second Expand
// after shlex. Yeah, crazy. I'm guessing this isn't secure, but just trying to avoid a foot gun.
part = os.Expand(part, func(s string) string {
return "${__" + s + "}"
})
}
parts = append(parts, part)
}

parts, err := shlex.Split(strings.Join(parts, ""))
if err != nil {
return nil, err
}

for i, part := range parts {
parts[i] = os.Expand(part, func(s string) string {
if strings.HasPrefix(s, "__") {
return "${" + s[2:] + "}"
}
return envMap[s]
return "${" + s + "}"
})
}

return parts, nil
}

// splitByQuotes will split a string by parsing matching double quotes (with \ as the escape character).
// The return value conforms to the following properties
// 1. s == strings.Join(result, "")
// 2. Even indexes are strings that were not in quotes.
// 3. Odd indexes are strings that were quoted.
//
// Example: s = `In a "quoted string" quotes can be escaped with \"`
//
// result = [`In a `, `"quoted string"`, ` quotes can be escaped with \"`]
func splitByQuotes(s string) (result []string) {
var (
buf strings.Builder
inEscape, inQuote bool
)

for _, c := range s {
if inEscape {
buf.WriteRune(c)
inEscape = false
continue
}

switch c {
case '"':
if inQuote {
buf.WriteRune(c)
}
result = append(result, buf.String())
buf.Reset()
if !inQuote {
buf.WriteRune(c)
}
inQuote = !inQuote
case '\\':
inEscape = true
buf.WriteRune(c)
default:
buf.WriteRune(c)
}
if runtime.GOOS == "windows" {
args[0] = strings.ReplaceAll(args[0], "/", "\\")
}

if buf.Len() > 0 {
if inQuote {
result = append(result, "")
}
result = append(result, buf.String())
if useShell {
args = append([]string{"/bin/sh", "-c"}, strings.Join(args, " "))
} else {
args[0] = env.Lookup(envvars, args[0])
}

return
cmd := exec.CommandContext(ctx, args[0], args[1:]...)
cmd.Env = compressEnv(envvars)
return cmd, stop, nil
}
135 changes: 0 additions & 135 deletions pkg/engine/cmd_test.go

This file was deleted.

1 change: 1 addition & 0 deletions pkg/engine/daemon.go
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@ func (e *Engine) startDaemon(tool types.Tool) (string, error) {
},
tool,
"{}",
false,
)
if err != nil {
return url, err
Expand Down
18 changes: 16 additions & 2 deletions pkg/repos/get.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ type Runtime interface {
ID() string
Supports(tool types.Tool, cmd []string) bool
Setup(ctx context.Context, tool types.Tool, dataRoot, toolSource string, env []string) ([]string, error)
GetHash(tool types.Tool) (string, error)
}

type noopRuntime struct {
Expand All @@ -37,6 +38,10 @@ func (n noopRuntime) ID() string {
return "none"
}

func (n noopRuntime) GetHash(_ types.Tool) (string, error) {
return "", nil
}

func (n noopRuntime) Supports(_ types.Tool, _ []string) bool {
return false
}
Expand Down Expand Up @@ -183,8 +188,13 @@ func (m *Manager) setup(ctx context.Context, runtime Runtime, tool types.Tool, e
locker.Lock(tool.ID)
defer locker.Unlock(tool.ID)

runtimeHash, err := runtime.GetHash(tool)
if err != nil {
return "", nil, err
}

target := filepath.Join(m.storageDir, tool.Source.Repo.Revision, tool.Source.Repo.Path, tool.Source.Repo.Name, runtime.ID())
targetFinal := filepath.Join(target, tool.Source.Repo.Path)
targetFinal := filepath.Join(target, tool.Source.Repo.Path+runtimeHash)
doneFile := targetFinal + ".done"
envData, err := os.ReadFile(doneFile)
if err == nil {
Expand Down Expand Up @@ -251,7 +261,11 @@ func (m *Manager) GetContext(ctx context.Context, tool types.Tool, cmd, env []st
for _, runtime := range m.runtimes {
if runtime.Supports(tool, cmd) {
log.Debugf("Runtime %s supports %v", runtime.ID(), cmd)
return m.setup(ctx, runtime, tool, env)
wd, env, err := m.setup(ctx, runtime, tool, env)
if isLocal {
wd = tool.WorkingDir
}
return wd, env, err
}
}

Expand Down
Loading

0 comments on commit c2c5d99

Please sign in to comment.