From 0f820879b595fe317a659dd53f67b8fd2826a958 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Sun, 2 Jun 2024 22:46:37 -0700 Subject: [PATCH 1/5] feat: add basic bash support for windows --- go.mod | 12 +-- go.sum | 24 ++--- pkg/engine/cmd.go | 5 + pkg/repos/download/extract.go | 14 +++ pkg/repos/get.go | 43 ++++++-- pkg/repos/runtimes/busybox/SHASUMS256.txt | 1 + pkg/repos/runtimes/busybox/busybox.go | 107 ++++++++++++++++++++ pkg/repos/runtimes/busybox/busybox_test.go | 41 ++++++++ pkg/repos/runtimes/busybox/log.go | 5 + pkg/repos/runtimes/default.go | 2 + pkg/tests/runner_test.go | 3 - pkg/tests/testdata/TestContextArg/other.gpt | 4 +- pkg/tests/testdata/TestContextArg/test.gpt | 2 +- pkg/tests/tester/runner.go | 12 ++- 14 files changed, 243 insertions(+), 32 deletions(-) create mode 100644 pkg/repos/runtimes/busybox/SHASUMS256.txt create mode 100644 pkg/repos/runtimes/busybox/busybox.go create mode 100644 pkg/repos/runtimes/busybox/busybox_test.go create mode 100644 pkg/repos/runtimes/busybox/log.go diff --git a/go.mod b/go.mod index 783e638b..c84dae97 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc golang.org/x/sync v0.7.0 - golang.org/x/term v0.20.0 + golang.org/x/term v0.22.0 gopkg.in/yaml.v3 v3.0.1 gotest.tools/v3 v3.5.1 sigs.k8s.io/yaml v1.4.0 @@ -108,10 +108,10 @@ require ( github.com/yuin/goldmark v1.5.4 // indirect github.com/yuin/goldmark-emoji v1.0.2 // indirect go4.org v0.0.0-20200411211856-f5505b9728dd // indirect - golang.org/x/mod v0.17.0 // indirect - golang.org/x/net v0.24.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect - golang.org/x/tools v0.20.0 // indirect + golang.org/x/mod v0.19.0 // indirect + golang.org/x/net v0.27.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/text v0.16.0 // indirect + golang.org/x/tools v0.23.0 // indirect mvdan.cc/gofumpt v0.6.0 // indirect ) diff --git a/go.sum b/go.sum index fd165d3f..7e2d7b75 100644 --- a/go.sum +++ b/go.sum @@ -396,8 +396,8 @@ golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= +golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -419,8 +419,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -474,8 +474,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -485,8 +485,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= -golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= -golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -498,8 +498,8 @@ golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -531,8 +531,8 @@ golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= -golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY= -golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= +golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= +golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/engine/cmd.go b/pkg/engine/cmd.go index d62aad2e..8e7b234e 100644 --- a/pkg/engine/cmd.go +++ b/pkg/engine/cmd.go @@ -10,6 +10,7 @@ import ( "io" "os" "os/exec" + "path" "path/filepath" "runtime" "sort" @@ -269,6 +270,10 @@ func (e *Engine) newCommand(ctx context.Context, extraEnv []string, tool types.T }) } + if runtime.GOOS == "windows" && (args[0] == "/bin/bash" || args[0] == "/bin/sh") { + args[0] = path.Base(args[0]) + } + if runtime.GOOS == "windows" && (args[0] == "/usr/bin/env" || args[0] == "/bin/env") { args = args[1:] } diff --git a/pkg/repos/download/extract.go b/pkg/repos/download/extract.go index 95a82d74..4cf09f0c 100644 --- a/pkg/repos/download/extract.go +++ b/pkg/repos/download/extract.go @@ -9,7 +9,9 @@ import ( "net/http" "net/url" "os" + "path" "path/filepath" + "strings" "time" "github.com/mholt/archiver/v4" @@ -60,6 +62,18 @@ func Extract(ctx context.Context, downloadURL, digest, targetDir string) error { return err } + bin := path.Base(parsedURL.Path) + if strings.HasSuffix(bin, ".exe") { + dst, err := os.Create(filepath.Join(targetDir, bin)) + if err != nil { + return err + } + defer dst.Close() + + _, err = io.Copy(dst, tmpFile) + return err + } + format, input, err := archiver.Identify(filepath.Base(parsedURL.Path), tmpFile) if err != nil { return err diff --git a/pkg/repos/get.go b/pkg/repos/get.go index 2f96d8b3..bead4a7a 100644 --- a/pkg/repos/get.go +++ b/pkg/repos/get.go @@ -15,6 +15,7 @@ import ( "github.com/BurntSushi/locker" "github.com/gptscript-ai/gptscript/pkg/config" "github.com/gptscript-ai/gptscript/pkg/credentials" + "github.com/gptscript-ai/gptscript/pkg/hash" "github.com/gptscript-ai/gptscript/pkg/loader/github" "github.com/gptscript-ai/gptscript/pkg/repos/git" "github.com/gptscript-ai/gptscript/pkg/repos/runtimes/golang" @@ -51,6 +52,7 @@ type Manager struct { credHelperDirs credentials.CredentialHelperDirs runtimes []Runtime credHelperConfig *credHelperConfig + supportLocal bool } type credHelperConfig struct { @@ -60,6 +62,10 @@ type credHelperConfig struct { env []string } +func (m *Manager) SetSupportLocal() { + m.supportLocal = true +} + func New(cacheDir string, runtimes ...Runtime) *Manager { root := filepath.Join(cacheDir, "repos") return &Manager{ @@ -200,8 +206,14 @@ func (m *Manager) setup(ctx context.Context, runtime Runtime, tool types.Tool, e _ = os.RemoveAll(doneFile) _ = os.RemoveAll(target) - if err := git.Checkout(ctx, m.gitDir, tool.Source.Repo.Root, tool.Source.Repo.Revision, target); err != nil { - return "", nil, err + if tool.Source.Repo.VCS == "git" { + if err := git.Checkout(ctx, m.gitDir, tool.Source.Repo.Root, tool.Source.Repo.Revision, target); err != nil { + return "", nil, err + } + } else { + if err := os.MkdirAll(target, 0755); err != nil { + return "", nil, err + } } newEnv, err := runtime.Setup(ctx, m.runtimeDir, targetFinal, env) @@ -227,12 +239,25 @@ func (m *Manager) setup(ctx context.Context, runtime Runtime, tool types.Tool, e } func (m *Manager) GetContext(ctx context.Context, tool types.Tool, cmd, env []string) (string, []string, error) { - if tool.Source.Repo == nil { - return tool.WorkingDir, env, nil - } + var isLocal bool + if !m.supportLocal { + if tool.Source.Repo == nil { + return tool.WorkingDir, env, nil + } - if tool.Source.Repo.VCS != "git" { - return "", nil, fmt.Errorf("only git is supported, found VCS %s for %s", tool.Source.Repo.VCS, tool.ID) + if tool.Source.Repo.VCS != "git" { + return "", nil, fmt.Errorf("only git is supported, found VCS %s for %s", tool.Source.Repo.VCS, tool.ID) + } + } else if tool.Source.Repo == nil { + isLocal = true + id := hash.Digest(tool)[:12] + tool.Source.Repo = &types.Repo{ + VCS: "", + Root: id, + Path: "/", + Name: id, + Revision: id, + } } for _, runtime := range m.runtimes { @@ -242,5 +267,9 @@ func (m *Manager) GetContext(ctx context.Context, tool types.Tool, cmd, env []st } } + if isLocal { + return tool.WorkingDir, env, nil + } + return m.setup(ctx, &noopRuntime{}, tool, env) } diff --git a/pkg/repos/runtimes/busybox/SHASUMS256.txt b/pkg/repos/runtimes/busybox/SHASUMS256.txt new file mode 100644 index 00000000..7da1aff6 --- /dev/null +++ b/pkg/repos/runtimes/busybox/SHASUMS256.txt @@ -0,0 +1 @@ +6d2dfd1c1412c3550a89071a1b36a6f6073844320e687343d1dfc72719ecb8d9 FRP-5301-gda71f7c57/busybox-w64-FRP-5301-gda71f7c57.exe \ No newline at end of file diff --git a/pkg/repos/runtimes/busybox/busybox.go b/pkg/repos/runtimes/busybox/busybox.go new file mode 100644 index 00000000..b0c00a0c --- /dev/null +++ b/pkg/repos/runtimes/busybox/busybox.go @@ -0,0 +1,107 @@ +package busybox + +import ( + "bufio" + "bytes" + "context" + _ "embed" + "errors" + "fmt" + "io/fs" + "os" + "os/exec" + "path" + "path/filepath" + "runtime" + "strings" + + runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env" + "github.com/gptscript-ai/gptscript/pkg/hash" + "github.com/gptscript-ai/gptscript/pkg/repos/download" +) + +//go:embed SHASUMS256.txt +var releasesData []byte + +const downloadURL = "https://github.com/gptscript-ai/busybox-w32/releases/download/%s" + +type Runtime struct { +} + +func (r *Runtime) ID() string { + return "busybox" +} + +func (r *Runtime) Supports(cmd []string) bool { + if runtime.GOOS != "windows" { + return false + } + for _, bin := range []string{"bash", "sh", "/bin/sh", "/bin/bash"} { + if runtimeEnv.Matches(cmd, bin) { + return true + } + } + return false +} + +func (r *Runtime) Setup(ctx context.Context, dataRoot, _ string, env []string) ([]string, error) { + binPath, err := r.getRuntime(ctx, dataRoot) + if err != nil { + return nil, err + } + + newEnv := runtimeEnv.AppendPath(env, binPath) + return newEnv, nil +} + +func (r *Runtime) getReleaseAndDigest() (string, string, error) { + scanner := bufio.NewScanner(bytes.NewReader(releasesData)) + for scanner.Scan() { + line := scanner.Text() + fields := strings.Fields(line) + return fmt.Sprintf(downloadURL, fields[1]), fields[0], nil + } + + return "", "", fmt.Errorf("failed to find %s release", r.ID()) +} + +func (r *Runtime) getRuntime(ctx context.Context, cwd string) (string, error) { + url, sha, err := r.getReleaseAndDigest() + if err != nil { + return "", err + } + + target := filepath.Join(cwd, "busybox", hash.ID(url, sha)) + if _, err := os.Stat(target); err == nil { + return target, nil + } else if !errors.Is(err, fs.ErrNotExist) { + return "", err + } + + log.Infof("Downloading Busybox") + tmp := target + ".download" + defer os.RemoveAll(tmp) + + if err := os.MkdirAll(tmp, 0755); err != nil { + return "", err + } + + if err := download.Extract(ctx, url, sha, tmp); err != nil { + return "", err + } + + bbExe := filepath.Join(tmp, path.Base(url)) + + cmd := exec.Command(bbExe, "--install", ".") + cmd.Dir = filepath.Dir(bbExe) + + if err := cmd.Run(); err != nil { + return "", err + } + + if err := os.Rename(tmp, target); err != nil { + return "", err + } + + return target, nil +} diff --git a/pkg/repos/runtimes/busybox/busybox_test.go b/pkg/repos/runtimes/busybox/busybox_test.go new file mode 100644 index 00000000..f3add18a --- /dev/null +++ b/pkg/repos/runtimes/busybox/busybox_test.go @@ -0,0 +1,41 @@ +package busybox + +import ( + "context" + "errors" + "io/fs" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/adrg/xdg" + "github.com/samber/lo" + "github.com/stretchr/testify/require" +) + +var ( + testCacheHome = lo.Must(xdg.CacheFile("gptscript-test-cache/runtime")) +) + +func firstPath(s []string) string { + _, p, _ := strings.Cut(s[0], "=") + return strings.Split(p, string(os.PathListSeparator))[0] +} + +func TestRuntime(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skip() + } + + r := Runtime{} + + s, err := r.Setup(context.Background(), testCacheHome, "testdata", os.Environ()) + require.NoError(t, err) + _, err = os.Stat(filepath.Join(firstPath(s), "busybox.exe")) + if errors.Is(err, fs.ErrNotExist) { + _, err = os.Stat(filepath.Join(firstPath(s), "busybox")) + } + require.NoError(t, err) +} diff --git a/pkg/repos/runtimes/busybox/log.go b/pkg/repos/runtimes/busybox/log.go new file mode 100644 index 00000000..b7e486f1 --- /dev/null +++ b/pkg/repos/runtimes/busybox/log.go @@ -0,0 +1,5 @@ +package busybox + +import "github.com/gptscript-ai/gptscript/pkg/mvl" + +var log = mvl.Package() diff --git a/pkg/repos/runtimes/default.go b/pkg/repos/runtimes/default.go index d37cca8f..3782e26e 100644 --- a/pkg/repos/runtimes/default.go +++ b/pkg/repos/runtimes/default.go @@ -3,12 +3,14 @@ package runtimes import ( "github.com/gptscript-ai/gptscript/pkg/engine" "github.com/gptscript-ai/gptscript/pkg/repos" + "github.com/gptscript-ai/gptscript/pkg/repos/runtimes/busybox" "github.com/gptscript-ai/gptscript/pkg/repos/runtimes/golang" "github.com/gptscript-ai/gptscript/pkg/repos/runtimes/node" "github.com/gptscript-ai/gptscript/pkg/repos/runtimes/python" ) var Runtimes = []repos.Runtime{ + &busybox.Runtime{}, &python.Runtime{ Version: "3.12", Default: true, diff --git a/pkg/tests/runner_test.go b/pkg/tests/runner_test.go index 12eff23a..60f1cfc3 100644 --- a/pkg/tests/runner_test.go +++ b/pkg/tests/runner_test.go @@ -751,9 +751,6 @@ func TestGlobalErr(t *testing.T) { } func TestContextArg(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip() - } runner := tester.NewRunner(t) x, err := runner.Run("", `{ "file": "foo.db" diff --git a/pkg/tests/testdata/TestContextArg/other.gpt b/pkg/tests/testdata/TestContextArg/other.gpt index b1acd66a..f97b4ba6 100644 --- a/pkg/tests/testdata/TestContextArg/other.gpt +++ b/pkg/tests/testdata/TestContextArg/other.gpt @@ -2,5 +2,5 @@ name: fromcontext args: first: an arg args: second: an arg -#!/bin/bash -echo this is from other context ${first} and then ${second} \ No newline at end of file +#!/usr/bin/env bash +echo this is from other context ${FIRST} and then ${SECOND} \ No newline at end of file diff --git a/pkg/tests/testdata/TestContextArg/test.gpt b/pkg/tests/testdata/TestContextArg/test.gpt index 9569aaf9..50d2ccf2 100644 --- a/pkg/tests/testdata/TestContextArg/test.gpt +++ b/pkg/tests/testdata/TestContextArg/test.gpt @@ -9,4 +9,4 @@ name: fromcontext args: first: an arg #!/bin/bash -echo this is from context -- ${first} \ No newline at end of file +echo this is from context -- ${FIRST} \ No newline at end of file diff --git a/pkg/tests/tester/runner.go b/pkg/tests/tester/runner.go index 775f0248..ef75c0f5 100644 --- a/pkg/tests/tester/runner.go +++ b/pkg/tests/tester/runner.go @@ -8,8 +8,11 @@ import ( "path/filepath" "testing" + "github.com/adrg/xdg" "github.com/gptscript-ai/gptscript/pkg/credentials" "github.com/gptscript-ai/gptscript/pkg/loader" + "github.com/gptscript-ai/gptscript/pkg/repos" + "github.com/gptscript-ai/gptscript/pkg/repos/runtimes" "github.com/gptscript-ai/gptscript/pkg/runner" "github.com/gptscript-ai/gptscript/pkg/types" "github.com/hexops/autogold/v2" @@ -171,8 +174,15 @@ func NewRunner(t *testing.T) *Runner { t: t, } + cacheDir, err := xdg.CacheFile("gptscript-test-cache/runtime") + require.NoError(t, err) + + rm := runtimes.Default(cacheDir) + rm.(*repos.Manager).SetSupportLocal() + run, err := runner.New(c, credentials.NoopStore{}, runner.Options{ - Sequential: true, + Sequential: true, + RuntimeManager: rm, }) require.NoError(t, err) From 4c3da3a352b5a5f26553685fb07938529d7ce766 Mon Sep 17 00:00:00 2001 From: Taylor Price Date: Fri, 2 Aug 2024 16:02:37 -0700 Subject: [PATCH 2/5] chore: uppercase env variable names Signed-off-by: Taylor Price --- integration/scripts/cred_scopes.gpt | 4 ++-- pkg/engine/cmd.go | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/integration/scripts/cred_scopes.gpt b/integration/scripts/cred_scopes.gpt index 7319f163..dc8e24e7 100644 --- a/integration/scripts/cred_scopes.gpt +++ b/integration/scripts/cred_scopes.gpt @@ -149,8 +149,8 @@ name: getcred import os import json -var = os.getenv('var') -val = os.getenv('val') +var = os.getenv('VAR') +val = os.getenv('VAL') output = { "env": { diff --git a/pkg/engine/cmd.go b/pkg/engine/cmd.go index 8e7b234e..869e67dc 100644 --- a/pkg/engine/cmd.go +++ b/pkg/engine/cmd.go @@ -204,10 +204,9 @@ var ignoreENV = map[string]struct{}{ } func appendEnv(envs []string, k, v string) []string { - for _, k := range []string{k, env.ToEnvLike(k)} { - if _, ignore := ignoreENV[k]; !ignore { - envs = append(envs, k+"="+v) - } + //fmt.Printf("%s=%s\n", k, v) + if _, ignore := ignoreENV[k]; !ignore { + envs = append(envs, strings.ToUpper(k)+"="+v) } return envs } From 9b8dc494ac29a921ea274b12814b2117514b655f Mon Sep 17 00:00:00 2001 From: Taylor Price Date: Mon, 5 Aug 2024 10:05:35 -0700 Subject: [PATCH 3/5] fix: correct small comment typo --- pkg/tests/smoke/smoke_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tests/smoke/smoke_test.go b/pkg/tests/smoke/smoke_test.go index 66374ef1..a6d0ab2c 100644 --- a/pkg/tests/smoke/smoke_test.go +++ b/pkg/tests/smoke/smoke_test.go @@ -83,7 +83,7 @@ func TestSmoke(t *testing.T) { actualEvents, ` - disregard differences in timestamps, generated IDs, natural language verbiage, and event order -- omit callProgress events from the comparision +- omit callProgress events from the comparison - the overall stream of events and set of tools called should roughly match - arguments passed in tool calls should be roughly the same - the final callFinish event should be semantically similar From ae328ac395ecfc710399a5b50fbc0aaa799c4168 Mon Sep 17 00:00:00 2001 From: Taylor Price Date: Mon, 5 Aug 2024 14:05:16 -0700 Subject: [PATCH 4/5] chore: ensure key is env-like Signed-off-by: Taylor Price --- pkg/engine/cmd.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/engine/cmd.go b/pkg/engine/cmd.go index 869e67dc..484b00cb 100644 --- a/pkg/engine/cmd.go +++ b/pkg/engine/cmd.go @@ -204,9 +204,8 @@ var ignoreENV = map[string]struct{}{ } func appendEnv(envs []string, k, v string) []string { - //fmt.Printf("%s=%s\n", k, v) if _, ignore := ignoreENV[k]; !ignore { - envs = append(envs, strings.ToUpper(k)+"="+v) + envs = append(envs, strings.ToUpper(env.ToEnvLike(k))+"="+v) } return envs } From 806fdc38b48658cb5bb68cb492f411f4fc1d55c0 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Mon, 5 Aug 2024 20:56:56 -0700 Subject: [PATCH 5/5] feat: support inline package.json and requirements.txt --- pkg/engine/cmd.go | 2 +- pkg/env/env.go | 9 ++ pkg/parser/parser.go | 40 +++++- pkg/parser/parser_test.go | 29 ++++ pkg/repos/get.go | 30 ++-- pkg/repos/runtimes/busybox/busybox.go | 5 +- pkg/repos/runtimes/busybox/busybox_test.go | 3 +- pkg/repos/runtimes/golang/golang.go | 8 +- pkg/repos/runtimes/golang/golang_test.go | 3 +- pkg/repos/runtimes/node/node.go | 26 +++- pkg/repos/runtimes/node/node_test.go | 5 +- pkg/repos/runtimes/python/python.go | 29 +++- pkg/repos/runtimes/python/python_test.go | 3 +- pkg/tests/runner_test.go | 21 +++ .../testdata/TestRuntimes/call1-resp.golden | 16 +++ pkg/tests/testdata/TestRuntimes/call1.golden | 37 +++++ .../testdata/TestRuntimes/call2-resp.golden | 16 +++ pkg/tests/testdata/TestRuntimes/call2.golden | 70 +++++++++ .../testdata/TestRuntimes/call3-resp.golden | 16 +++ pkg/tests/testdata/TestRuntimes/call3.golden | 103 +++++++++++++ .../testdata/TestRuntimes/call4-resp.golden | 9 ++ pkg/tests/testdata/TestRuntimes/call4.golden | 136 ++++++++++++++++++ pkg/tests/testdata/TestRuntimes/test.gpt | 58 ++++++++ pkg/tests/tester/runner.go | 2 - pkg/types/tool.go | 5 + 25 files changed, 635 insertions(+), 46 deletions(-) create mode 100644 pkg/tests/testdata/TestRuntimes/call1-resp.golden create mode 100644 pkg/tests/testdata/TestRuntimes/call1.golden create mode 100644 pkg/tests/testdata/TestRuntimes/call2-resp.golden create mode 100644 pkg/tests/testdata/TestRuntimes/call2.golden create mode 100644 pkg/tests/testdata/TestRuntimes/call3-resp.golden create mode 100644 pkg/tests/testdata/TestRuntimes/call3.golden create mode 100644 pkg/tests/testdata/TestRuntimes/call4-resp.golden create mode 100644 pkg/tests/testdata/TestRuntimes/call4.golden create mode 100644 pkg/tests/testdata/TestRuntimes/test.gpt diff --git a/pkg/engine/cmd.go b/pkg/engine/cmd.go index 484b00cb..57dfd477 100644 --- a/pkg/engine/cmd.go +++ b/pkg/engine/cmd.go @@ -282,7 +282,7 @@ func (e *Engine) newCommand(ctx context.Context, extraEnv []string, tool types.T ) if strings.TrimSpace(rest) != "" { - f, err := os.CreateTemp("", version.ProgramName+requiredFileExtensions[args[0]]) + f, err := os.CreateTemp(env.Getenv("GPTSCRIPT_TMPDIR", envvars), version.ProgramName+requiredFileExtensions[args[0]]) if err != nil { return nil, nil, err } diff --git a/pkg/env/env.go b/pkg/env/env.go index bedd5f9d..2994825b 100644 --- a/pkg/env/env.go +++ b/pkg/env/env.go @@ -26,6 +26,15 @@ func ToEnvLike(v string) string { return strings.ToUpper(v) } +func Getenv(key string, envs []string) string { + for i := len(envs) - 1; i >= 0; i-- { + if k, v, ok := strings.Cut(envs[i], "="); ok && k == key { + return v + } + } + return "" +} + func Matches(cmd []string, bin string) bool { switch len(cmd) { case 0: diff --git a/pkg/parser/parser.go b/pkg/parser/parser.go index ff5d1374..b57cb658 100644 --- a/pkg/parser/parser.go +++ b/pkg/parser/parser.go @@ -16,7 +16,7 @@ import ( var ( sepRegex = regexp.MustCompile(`^\s*---+\s*$`) strictSepRegex = regexp.MustCompile(`^---\n$`) - skipRegex = regexp.MustCompile(`^![-\w]+\s*$`) + skipRegex = regexp.MustCompile(`^![-.:\w]+\s*$`) ) func normalize(key string) string { @@ -308,6 +308,8 @@ func Parse(input io.Reader, opts ...Options) (Document, error) { } } + nodes = assignMetadata(nodes) + if !opt.AssignGlobals { return Document{ Nodes: nodes, @@ -359,6 +361,42 @@ func Parse(input io.Reader, opts ...Options) (Document, error) { }, nil } +func assignMetadata(nodes []Node) (result []Node) { + metadata := map[string]map[string]string{} + result = make([]Node, 0, len(nodes)) + for _, node := range nodes { + if node.TextNode != nil { + body, ok := strings.CutPrefix(node.TextNode.Text, "!metadata:") + if ok { + line, rest, ok := strings.Cut(body, "\n") + if ok { + toolName, metaKey, ok := strings.Cut(strings.TrimSpace(line), ":") + if ok { + d, ok := metadata[toolName] + if !ok { + d = map[string]string{} + metadata[toolName] = d + } + d[metaKey] = strings.TrimSpace(rest) + } + } + } + } + } + if len(metadata) == 0 { + return nodes + } + + for _, node := range nodes { + if node.ToolNode != nil { + node.ToolNode.Tool.MetaData = metadata[node.ToolNode.Tool.Name] + } + result = append(result, node) + } + + return +} + func isGPTScriptHashBang(line string) bool { if !strings.HasPrefix(line, "#!") { return false diff --git a/pkg/parser/parser_test.go b/pkg/parser/parser_test.go index 9f682efa..3967ebd5 100644 --- a/pkg/parser/parser_test.go +++ b/pkg/parser/parser_test.go @@ -6,6 +6,7 @@ import ( "github.com/gptscript-ai/gptscript/pkg/types" "github.com/hexops/autogold/v2" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -239,3 +240,31 @@ share output filters: shared }}, }}).Equal(t, out) } + +func TestParseMetaData(t *testing.T) { + input := ` +name: first + +body +--- +!metadata:first:package.json +foo=base +f + +--- +!metadata:first2:requirements.txt +asdf2 + +--- +!metadata:first:requirements.txt +asdf +` + tools, err := ParseTools(strings.NewReader(input)) + require.NoError(t, err) + + assert.Len(t, tools, 1) + autogold.Expect(map[string]string{ + "package.json": "foo=base\nf", + "requirements.txt": "asdf", + }).Equal(t, tools[0].MetaData) +} diff --git a/pkg/repos/get.go b/pkg/repos/get.go index bead4a7a..fc675c58 100644 --- a/pkg/repos/get.go +++ b/pkg/repos/get.go @@ -26,8 +26,8 @@ const credentialHelpersRepo = "github.com/gptscript-ai/gptscript-credential-help type Runtime interface { ID() string - Supports(cmd []string) bool - Setup(ctx context.Context, dataRoot, toolSource string, env []string) ([]string, error) + Supports(tool types.Tool, cmd []string) bool + Setup(ctx context.Context, tool types.Tool, dataRoot, toolSource string, env []string) ([]string, error) } type noopRuntime struct { @@ -37,11 +37,11 @@ func (n noopRuntime) ID() string { return "none" } -func (n noopRuntime) Supports(_ []string) bool { +func (n noopRuntime) Supports(_ types.Tool, _ []string) bool { return false } -func (n noopRuntime) Setup(_ context.Context, _, _ string, _ []string) ([]string, error) { +func (n noopRuntime) Setup(_ context.Context, _ types.Tool, _, _ string, _ []string) ([]string, error) { return nil, nil } @@ -52,7 +52,6 @@ type Manager struct { credHelperDirs credentials.CredentialHelperDirs runtimes []Runtime credHelperConfig *credHelperConfig - supportLocal bool } type credHelperConfig struct { @@ -62,10 +61,6 @@ type credHelperConfig struct { env []string } -func (m *Manager) SetSupportLocal() { - m.supportLocal = true -} - func New(cacheDir string, runtimes ...Runtime) *Manager { root := filepath.Join(cacheDir, "repos") return &Manager{ @@ -216,7 +211,7 @@ func (m *Manager) setup(ctx context.Context, runtime Runtime, tool types.Tool, e } } - newEnv, err := runtime.Setup(ctx, m.runtimeDir, targetFinal, env) + newEnv, err := runtime.Setup(ctx, tool, m.runtimeDir, targetFinal, env) if err != nil { return "", nil, err } @@ -240,17 +235,10 @@ func (m *Manager) setup(ctx context.Context, runtime Runtime, tool types.Tool, e func (m *Manager) GetContext(ctx context.Context, tool types.Tool, cmd, env []string) (string, []string, error) { var isLocal bool - if !m.supportLocal { - if tool.Source.Repo == nil { - return tool.WorkingDir, env, nil - } - - if tool.Source.Repo.VCS != "git" { - return "", nil, fmt.Errorf("only git is supported, found VCS %s for %s", tool.Source.Repo.VCS, tool.ID) - } - } else if tool.Source.Repo == nil { + if tool.Source.Repo == nil { isLocal = true - id := hash.Digest(tool)[:12] + d, _ := json.Marshal(tool) + id := hash.Digest(d)[:12] tool.Source.Repo = &types.Repo{ VCS: "", Root: id, @@ -261,7 +249,7 @@ func (m *Manager) GetContext(ctx context.Context, tool types.Tool, cmd, env []st } for _, runtime := range m.runtimes { - if runtime.Supports(cmd) { + if runtime.Supports(tool, cmd) { log.Debugf("Runtime %s supports %v", runtime.ID(), cmd) return m.setup(ctx, runtime, tool, env) } diff --git a/pkg/repos/runtimes/busybox/busybox.go b/pkg/repos/runtimes/busybox/busybox.go index b0c00a0c..542ba94a 100644 --- a/pkg/repos/runtimes/busybox/busybox.go +++ b/pkg/repos/runtimes/busybox/busybox.go @@ -18,6 +18,7 @@ import ( runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env" "github.com/gptscript-ai/gptscript/pkg/hash" "github.com/gptscript-ai/gptscript/pkg/repos/download" + "github.com/gptscript-ai/gptscript/pkg/types" ) //go:embed SHASUMS256.txt @@ -32,7 +33,7 @@ func (r *Runtime) ID() string { return "busybox" } -func (r *Runtime) Supports(cmd []string) bool { +func (r *Runtime) Supports(_ types.Tool, cmd []string) bool { if runtime.GOOS != "windows" { return false } @@ -44,7 +45,7 @@ func (r *Runtime) Supports(cmd []string) bool { return false } -func (r *Runtime) Setup(ctx context.Context, dataRoot, _ string, env []string) ([]string, error) { +func (r *Runtime) Setup(ctx context.Context, _ types.Tool, dataRoot, _ string, env []string) ([]string, error) { binPath, err := r.getRuntime(ctx, dataRoot) if err != nil { return nil, err diff --git a/pkg/repos/runtimes/busybox/busybox_test.go b/pkg/repos/runtimes/busybox/busybox_test.go index f3add18a..77bfae59 100644 --- a/pkg/repos/runtimes/busybox/busybox_test.go +++ b/pkg/repos/runtimes/busybox/busybox_test.go @@ -11,6 +11,7 @@ import ( "testing" "github.com/adrg/xdg" + "github.com/gptscript-ai/gptscript/pkg/types" "github.com/samber/lo" "github.com/stretchr/testify/require" ) @@ -31,7 +32,7 @@ func TestRuntime(t *testing.T) { r := Runtime{} - s, err := r.Setup(context.Background(), testCacheHome, "testdata", os.Environ()) + s, err := r.Setup(context.Background(), types.Tool{}, testCacheHome, "testdata", os.Environ()) require.NoError(t, err) _, err = os.Stat(filepath.Join(firstPath(s), "busybox.exe")) if errors.Is(err, fs.ErrNotExist) { diff --git a/pkg/repos/runtimes/golang/golang.go b/pkg/repos/runtimes/golang/golang.go index 28300439..b19cfe90 100644 --- a/pkg/repos/runtimes/golang/golang.go +++ b/pkg/repos/runtimes/golang/golang.go @@ -18,6 +18,7 @@ import ( runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env" "github.com/gptscript-ai/gptscript/pkg/hash" "github.com/gptscript-ai/gptscript/pkg/repos/download" + "github.com/gptscript-ai/gptscript/pkg/types" ) //go:embed digests.txt @@ -34,11 +35,12 @@ func (r *Runtime) ID() string { return "go" + r.Version } -func (r *Runtime) Supports(cmd []string) bool { - return len(cmd) > 0 && cmd[0] == "${GPTSCRIPT_TOOL_DIR}/bin/gptscript-go-tool" +func (r *Runtime) Supports(tool types.Tool, cmd []string) bool { + return tool.Source.IsGit() && + len(cmd) > 0 && cmd[0] == "${GPTSCRIPT_TOOL_DIR}/bin/gptscript-go-tool" } -func (r *Runtime) Setup(ctx context.Context, dataRoot, toolSource string, env []string) ([]string, error) { +func (r *Runtime) Setup(ctx context.Context, _ types.Tool, dataRoot, toolSource string, env []string) ([]string, error) { binPath, err := r.getRuntime(ctx, dataRoot) if err != nil { return nil, err diff --git a/pkg/repos/runtimes/golang/golang_test.go b/pkg/repos/runtimes/golang/golang_test.go index 5f71fb50..56098a51 100644 --- a/pkg/repos/runtimes/golang/golang_test.go +++ b/pkg/repos/runtimes/golang/golang_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/adrg/xdg" + "github.com/gptscript-ai/gptscript/pkg/types" "github.com/samber/lo" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -27,7 +28,7 @@ func TestRuntime(t *testing.T) { Version: "1.22.1", } - s, err := r.Setup(context.Background(), testCacheHome, "testdata", os.Environ()) + s, err := r.Setup(context.Background(), types.Tool{}, testCacheHome, "testdata", os.Environ()) require.NoError(t, err) p, v, _ := strings.Cut(s[0], "=") v, _, _ = strings.Cut(v, string(filepath.ListSeparator)) diff --git a/pkg/repos/runtimes/node/node.go b/pkg/repos/runtimes/node/node.go index 575e3b23..fde5103d 100644 --- a/pkg/repos/runtimes/node/node.go +++ b/pkg/repos/runtimes/node/node.go @@ -17,12 +17,16 @@ import ( runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env" "github.com/gptscript-ai/gptscript/pkg/hash" "github.com/gptscript-ai/gptscript/pkg/repos/download" + "github.com/gptscript-ai/gptscript/pkg/types" ) //go:embed SHASUMS256.txt.asc var releasesData []byte -const downloadURL = "https://nodejs.org/dist/%s/" +const ( + downloadURL = "https://nodejs.org/dist/%s/" + packageJSON = "package.json" +) type Runtime struct { // version something like "3.12" @@ -35,7 +39,10 @@ func (r *Runtime) ID() string { return "node" + r.Version } -func (r *Runtime) Supports(cmd []string) bool { +func (r *Runtime) Supports(tool types.Tool, cmd []string) bool { + if _, hasPackageJSON := tool.MetaData[packageJSON]; !hasPackageJSON && !tool.Source.IsGit() { + return false + } for _, testCmd := range []string{"node", "npx", "npm"} { if r.supports(testCmd, cmd) { return true @@ -54,17 +61,21 @@ func (r *Runtime) supports(testCmd string, cmd []string) bool { return runtimeEnv.Matches(cmd, testCmd) } -func (r *Runtime) Setup(ctx context.Context, dataRoot, toolSource string, env []string) ([]string, error) { +func (r *Runtime) Setup(ctx context.Context, tool types.Tool, dataRoot, toolSource string, env []string) ([]string, error) { binPath, err := r.getRuntime(ctx, dataRoot) if err != nil { return nil, err } newEnv := runtimeEnv.AppendPath(env, binPath) - if err := r.runNPM(ctx, toolSource, binPath, append(env, newEnv...)); err != nil { + if err := r.runNPM(ctx, tool, toolSource, binPath, append(env, newEnv...)); err != nil { return nil, err } + if _, ok := tool.MetaData[packageJSON]; ok { + newEnv = append(newEnv, "GPTSCRIPT_TMPDIR="+toolSource) + } + return newEnv, nil } @@ -100,11 +111,16 @@ func (r *Runtime) getReleaseAndDigest() (string, string, error) { return "", "", fmt.Errorf("failed to find %s release for os=%s arch=%s", r.ID(), osName(), arch()) } -func (r *Runtime) runNPM(ctx context.Context, toolSource, binDir string, env []string) error { +func (r *Runtime) runNPM(ctx context.Context, tool types.Tool, toolSource, binDir string, env []string) error { log.InfofCtx(ctx, "Running npm in %s", toolSource) cmd := debugcmd.New(ctx, filepath.Join(binDir, "npm"), "install") cmd.Env = env cmd.Dir = toolSource + if contents, ok := tool.MetaData[packageJSON]; ok { + if err := os.WriteFile(filepath.Join(toolSource, packageJSON), []byte(contents+"\n"), 0644); err != nil { + return err + } + } return cmd.Run() } diff --git a/pkg/repos/runtimes/node/node_test.go b/pkg/repos/runtimes/node/node_test.go index 50ef1e0a..014619c8 100644 --- a/pkg/repos/runtimes/node/node_test.go +++ b/pkg/repos/runtimes/node/node_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/adrg/xdg" + "github.com/gptscript-ai/gptscript/pkg/types" "github.com/samber/lo" "github.com/stretchr/testify/require" ) @@ -28,7 +29,7 @@ func TestRuntime(t *testing.T) { Version: "20", } - s, err := r.Setup(context.Background(), testCacheHome, "testdata", os.Environ()) + s, err := r.Setup(context.Background(), types.Tool{}, testCacheHome, "testdata", os.Environ()) require.NoError(t, err) _, err = os.Stat(filepath.Join(firstPath(s), "node.exe")) if errors.Is(err, fs.ErrNotExist) { @@ -42,7 +43,7 @@ func TestRuntime21(t *testing.T) { Version: "21", } - s, err := r.Setup(context.Background(), testCacheHome, "testdata", os.Environ()) + s, err := r.Setup(context.Background(), types.Tool{}, testCacheHome, "testdata", os.Environ()) require.NoError(t, err) _, err = os.Stat(filepath.Join(firstPath(s), "node.exe")) if errors.Is(err, fs.ErrNotExist) { diff --git a/pkg/repos/runtimes/python/python.go b/pkg/repos/runtimes/python/python.go index c031cb16..ae24f92a 100644 --- a/pkg/repos/runtimes/python/python.go +++ b/pkg/repos/runtimes/python/python.go @@ -17,12 +17,16 @@ import ( runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env" "github.com/gptscript-ai/gptscript/pkg/hash" "github.com/gptscript-ai/gptscript/pkg/repos/download" + "github.com/gptscript-ai/gptscript/pkg/types" ) //go:embed python.json var releasesData []byte -const uvVersion = "uv==0.2.27" +const ( + uvVersion = "uv==0.2.33" + requirementsTxt = "requirements.txt" +) type Release struct { OS string `json:"os,omitempty"` @@ -43,7 +47,10 @@ func (r *Runtime) ID() string { return "python" + r.Version } -func (r *Runtime) Supports(cmd []string) bool { +func (r *Runtime) Supports(tool types.Tool, cmd []string) bool { + if _, hasRequirements := tool.MetaData[requirementsTxt]; !hasRequirements && !tool.Source.IsGit() { + return false + } if runtimeEnv.Matches(cmd, r.ID()) { return true } @@ -112,7 +119,7 @@ func (r *Runtime) copyPythonForWindows(binDir string) error { return nil } -func (r *Runtime) Setup(ctx context.Context, dataRoot, toolSource string, env []string) ([]string, error) { +func (r *Runtime) Setup(ctx context.Context, tool types.Tool, dataRoot, toolSource string, env []string) ([]string, error) { binPath, err := r.getRuntime(ctx, dataRoot) if err != nil { return nil, err @@ -145,7 +152,7 @@ func (r *Runtime) Setup(ctx context.Context, dataRoot, toolSource string, env [] } } - if err := r.runPip(ctx, toolSource, binPath, append(env, newEnv...)); err != nil { + if err := r.runPip(ctx, tool, toolSource, binPath, append(env, newEnv...)); err != nil { return nil, err } @@ -170,9 +177,19 @@ func (r *Runtime) getReleaseAndDigest() (string, string, error) { return "", "", fmt.Errorf("failed to find an python runtime for %s", r.Version) } -func (r *Runtime) runPip(ctx context.Context, toolSource, binDir string, env []string) error { +func (r *Runtime) runPip(ctx context.Context, tool types.Tool, toolSource, binDir string, env []string) error { log.InfofCtx(ctx, "Running pip in %s", toolSource) - for _, req := range []string{"requirements-gptscript.txt", "requirements.txt"} { + if content, ok := tool.MetaData[requirementsTxt]; ok { + reqFile := filepath.Join(toolSource, requirementsTxt) + if err := os.WriteFile(reqFile, []byte(content+"\n"), 0644); err != nil { + return err + } + cmd := debugcmd.New(ctx, uvBin(binDir), "pip", "install", "-r", reqFile) + cmd.Env = env + return cmd.Run() + } + + for _, req := range []string{"requirements-gptscript.txt", requirementsTxt} { reqFile := filepath.Join(toolSource, req) if s, err := os.Stat(reqFile); err == nil && !s.IsDir() { cmd := debugcmd.New(ctx, uvBin(binDir), "pip", "install", "-r", reqFile) diff --git a/pkg/repos/runtimes/python/python_test.go b/pkg/repos/runtimes/python/python_test.go index fc2ededc..0e483305 100644 --- a/pkg/repos/runtimes/python/python_test.go +++ b/pkg/repos/runtimes/python/python_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/adrg/xdg" + "github.com/gptscript-ai/gptscript/pkg/types" "github.com/samber/lo" "github.com/stretchr/testify/require" ) @@ -27,7 +28,7 @@ func TestRuntime(t *testing.T) { Version: "3.12", } - s, err := r.Setup(context.Background(), testCacheHome, "testdata", os.Environ()) + s, err := r.Setup(context.Background(), types.Tool{}, testCacheHome, "testdata", os.Environ()) require.NoError(t, err) _, err = os.Stat(filepath.Join(firstPath(s), "python.exe")) if errors.Is(err, os.ErrNotExist) { diff --git a/pkg/tests/runner_test.go b/pkg/tests/runner_test.go index 60f1cfc3..424c84c1 100644 --- a/pkg/tests/runner_test.go +++ b/pkg/tests/runner_test.go @@ -997,3 +997,24 @@ func TestToolRefAll(t *testing.T) { r := tester.NewRunner(t) r.RunDefault() } + +func TestRuntimes(t *testing.T) { + r := tester.NewRunner(t) + r.RespondWith(tester.Result{ + Func: types.CompletionFunctionCall{ + Name: "py", + Arguments: "{}", + }, + }, tester.Result{ + Func: types.CompletionFunctionCall{ + Name: "node", + Arguments: "{}", + }, + }, tester.Result{ + Func: types.CompletionFunctionCall{ + Name: "bash", + Arguments: "{}", + }, + }) + r.RunDefault() +} diff --git a/pkg/tests/testdata/TestRuntimes/call1-resp.golden b/pkg/tests/testdata/TestRuntimes/call1-resp.golden new file mode 100644 index 00000000..1d53670a --- /dev/null +++ b/pkg/tests/testdata/TestRuntimes/call1-resp.golden @@ -0,0 +1,16 @@ +`{ + "role": "assistant", + "content": [ + { + "toolCall": { + "index": 0, + "id": "call_1", + "function": { + "name": "py", + "arguments": "{}" + } + } + } + ], + "usage": {} +}` diff --git a/pkg/tests/testdata/TestRuntimes/call1.golden b/pkg/tests/testdata/TestRuntimes/call1.golden new file mode 100644 index 00000000..67c7d9f7 --- /dev/null +++ b/pkg/tests/testdata/TestRuntimes/call1.golden @@ -0,0 +1,37 @@ +`{ + "model": "gpt-4o", + "tools": [ + { + "function": { + "toolID": "testdata/TestRuntimes/test.gpt:py", + "name": "py", + "parameters": null + } + }, + { + "function": { + "toolID": "testdata/TestRuntimes/test.gpt:node", + "name": "node", + "parameters": null + } + }, + { + "function": { + "toolID": "testdata/TestRuntimes/test.gpt:bash", + "name": "bash", + "parameters": null + } + } + ], + "messages": [ + { + "role": "system", + "content": [ + { + "text": "Dummy" + } + ], + "usage": {} + } + ] +}` diff --git a/pkg/tests/testdata/TestRuntimes/call2-resp.golden b/pkg/tests/testdata/TestRuntimes/call2-resp.golden new file mode 100644 index 00000000..4806793c --- /dev/null +++ b/pkg/tests/testdata/TestRuntimes/call2-resp.golden @@ -0,0 +1,16 @@ +`{ + "role": "assistant", + "content": [ + { + "toolCall": { + "index": 1, + "id": "call_2", + "function": { + "name": "node", + "arguments": "{}" + } + } + } + ], + "usage": {} +}` diff --git a/pkg/tests/testdata/TestRuntimes/call2.golden b/pkg/tests/testdata/TestRuntimes/call2.golden new file mode 100644 index 00000000..da456a5d --- /dev/null +++ b/pkg/tests/testdata/TestRuntimes/call2.golden @@ -0,0 +1,70 @@ +`{ + "model": "gpt-4o", + "tools": [ + { + "function": { + "toolID": "testdata/TestRuntimes/test.gpt:py", + "name": "py", + "parameters": null + } + }, + { + "function": { + "toolID": "testdata/TestRuntimes/test.gpt:node", + "name": "node", + "parameters": null + } + }, + { + "function": { + "toolID": "testdata/TestRuntimes/test.gpt:bash", + "name": "bash", + "parameters": null + } + } + ], + "messages": [ + { + "role": "system", + "content": [ + { + "text": "Dummy" + } + ], + "usage": {} + }, + { + "role": "assistant", + "content": [ + { + "toolCall": { + "index": 0, + "id": "call_1", + "function": { + "name": "py", + "arguments": "{}" + } + } + } + ], + "usage": {} + }, + { + "role": "tool", + "content": [ + { + "text": "py worked\r\n" + } + ], + "toolCall": { + "index": 0, + "id": "call_1", + "function": { + "name": "py", + "arguments": "{}" + } + }, + "usage": {} + } + ] +}` diff --git a/pkg/tests/testdata/TestRuntimes/call3-resp.golden b/pkg/tests/testdata/TestRuntimes/call3-resp.golden new file mode 100644 index 00000000..1103f824 --- /dev/null +++ b/pkg/tests/testdata/TestRuntimes/call3-resp.golden @@ -0,0 +1,16 @@ +`{ + "role": "assistant", + "content": [ + { + "toolCall": { + "index": 2, + "id": "call_3", + "function": { + "name": "bash", + "arguments": "{}" + } + } + } + ], + "usage": {} +}` diff --git a/pkg/tests/testdata/TestRuntimes/call3.golden b/pkg/tests/testdata/TestRuntimes/call3.golden new file mode 100644 index 00000000..f0792540 --- /dev/null +++ b/pkg/tests/testdata/TestRuntimes/call3.golden @@ -0,0 +1,103 @@ +`{ + "model": "gpt-4o", + "tools": [ + { + "function": { + "toolID": "testdata/TestRuntimes/test.gpt:py", + "name": "py", + "parameters": null + } + }, + { + "function": { + "toolID": "testdata/TestRuntimes/test.gpt:node", + "name": "node", + "parameters": null + } + }, + { + "function": { + "toolID": "testdata/TestRuntimes/test.gpt:bash", + "name": "bash", + "parameters": null + } + } + ], + "messages": [ + { + "role": "system", + "content": [ + { + "text": "Dummy" + } + ], + "usage": {} + }, + { + "role": "assistant", + "content": [ + { + "toolCall": { + "index": 0, + "id": "call_1", + "function": { + "name": "py", + "arguments": "{}" + } + } + } + ], + "usage": {} + }, + { + "role": "tool", + "content": [ + { + "text": "py worked\r\n" + } + ], + "toolCall": { + "index": 0, + "id": "call_1", + "function": { + "name": "py", + "arguments": "{}" + } + }, + "usage": {} + }, + { + "role": "assistant", + "content": [ + { + "toolCall": { + "index": 1, + "id": "call_2", + "function": { + "name": "node", + "arguments": "{}" + } + } + } + ], + "usage": {} + }, + { + "role": "tool", + "content": [ + { + "text": "node worked\n" + } + ], + "toolCall": { + "index": 1, + "id": "call_2", + "function": { + "name": "node", + "arguments": "{}" + } + }, + "usage": {} + } + ] +}` diff --git a/pkg/tests/testdata/TestRuntimes/call4-resp.golden b/pkg/tests/testdata/TestRuntimes/call4-resp.golden new file mode 100644 index 00000000..8135a8c9 --- /dev/null +++ b/pkg/tests/testdata/TestRuntimes/call4-resp.golden @@ -0,0 +1,9 @@ +`{ + "role": "assistant", + "content": [ + { + "text": "TEST RESULT CALL: 4" + } + ], + "usage": {} +}` diff --git a/pkg/tests/testdata/TestRuntimes/call4.golden b/pkg/tests/testdata/TestRuntimes/call4.golden new file mode 100644 index 00000000..04ac31f5 --- /dev/null +++ b/pkg/tests/testdata/TestRuntimes/call4.golden @@ -0,0 +1,136 @@ +`{ + "model": "gpt-4o", + "tools": [ + { + "function": { + "toolID": "testdata/TestRuntimes/test.gpt:py", + "name": "py", + "parameters": null + } + }, + { + "function": { + "toolID": "testdata/TestRuntimes/test.gpt:node", + "name": "node", + "parameters": null + } + }, + { + "function": { + "toolID": "testdata/TestRuntimes/test.gpt:bash", + "name": "bash", + "parameters": null + } + } + ], + "messages": [ + { + "role": "system", + "content": [ + { + "text": "Dummy" + } + ], + "usage": {} + }, + { + "role": "assistant", + "content": [ + { + "toolCall": { + "index": 0, + "id": "call_1", + "function": { + "name": "py", + "arguments": "{}" + } + } + } + ], + "usage": {} + }, + { + "role": "tool", + "content": [ + { + "text": "py worked\r\n" + } + ], + "toolCall": { + "index": 0, + "id": "call_1", + "function": { + "name": "py", + "arguments": "{}" + } + }, + "usage": {} + }, + { + "role": "assistant", + "content": [ + { + "toolCall": { + "index": 1, + "id": "call_2", + "function": { + "name": "node", + "arguments": "{}" + } + } + } + ], + "usage": {} + }, + { + "role": "tool", + "content": [ + { + "text": "node worked\n" + } + ], + "toolCall": { + "index": 1, + "id": "call_2", + "function": { + "name": "node", + "arguments": "{}" + } + }, + "usage": {} + }, + { + "role": "assistant", + "content": [ + { + "toolCall": { + "index": 2, + "id": "call_3", + "function": { + "name": "bash", + "arguments": "{}" + } + } + } + ], + "usage": {} + }, + { + "role": "tool", + "content": [ + { + "text": "bash works\n" + } + ], + "toolCall": { + "index": 2, + "id": "call_3", + "function": { + "name": "bash", + "arguments": "{}" + } + }, + "usage": {} + } + ] +}` diff --git a/pkg/tests/testdata/TestRuntimes/test.gpt b/pkg/tests/testdata/TestRuntimes/test.gpt new file mode 100644 index 00000000..db3ede64 --- /dev/null +++ b/pkg/tests/testdata/TestRuntimes/test.gpt @@ -0,0 +1,58 @@ +name: first +tools: py, node, bash + +Dummy + +--- +name: py + +#!/usr/bin/env python3 + +import requests +import platform + +# this is dumb hack to get the line endings to always be \r\n so the golden files match +# on both linux and windows +if platform.system() == 'Windows': + print('py worked') +else: + print('py worked\r') + +--- +!metadata:py:requirements.txt + +requests + +--- +name: node + +#!/usr/bin/env node + +import chalk from 'chalk'; +console.log("node worked") + +--- +!metadata:node:package.json + +{ + "name": "chalk-example", + "version": "1.0.0", + "type": "module", + "description": "A simple example project to demonstrate the use of chalk", + "main": "example.js", + "scripts": { + "start": "node example.js" + }, + "author": "Your Name", + "license": "MIT", + "dependencies": { + "chalk": "^5.0.0" + } + } + +--- +name: bash + +#!/bin/bash + +echo bash works \ No newline at end of file diff --git a/pkg/tests/tester/runner.go b/pkg/tests/tester/runner.go index ef75c0f5..a36c5e91 100644 --- a/pkg/tests/tester/runner.go +++ b/pkg/tests/tester/runner.go @@ -11,7 +11,6 @@ import ( "github.com/adrg/xdg" "github.com/gptscript-ai/gptscript/pkg/credentials" "github.com/gptscript-ai/gptscript/pkg/loader" - "github.com/gptscript-ai/gptscript/pkg/repos" "github.com/gptscript-ai/gptscript/pkg/repos/runtimes" "github.com/gptscript-ai/gptscript/pkg/runner" "github.com/gptscript-ai/gptscript/pkg/types" @@ -178,7 +177,6 @@ func NewRunner(t *testing.T) *Runner { require.NoError(t, err) rm := runtimes.Default(cacheDir) - rm.(*repos.Manager).SetSupportLocal() run, err := runner.New(c, credentials.NoopStore{}, runner.Options{ Sequential: true, diff --git a/pkg/types/tool.go b/pkg/types/tool.go index 54d5d817..61a67fa4 100644 --- a/pkg/types/tool.go +++ b/pkg/types/tool.go @@ -166,6 +166,7 @@ type Tool struct { ID string `json:"id,omitempty"` ToolMapping map[string][]ToolReference `json:"toolMapping,omitempty"` + MetaData map[string]string `json:"metaData,omitempty"` LocalTools map[string]string `json:"localTools,omitempty"` Source ToolSource `json:"source,omitempty"` WorkingDir string `json:"workingDir,omitempty"` @@ -793,6 +794,10 @@ type ToolSource struct { Repo *Repo `json:"repo,omitempty"` } +func (t ToolSource) IsGit() bool { + return t.Repo != nil && t.Repo.VCS == "git" +} + func (t ToolSource) String() string { return fmt.Sprintf("%s:%d", t.Location, t.LineNo) }