Merge pull request #692 from drpebcak/win-bash

go.mod

@@ -33,7 +33,7 @@ require (
 	github.com/xeipuuv/gojsonschema v1.2.0
 	golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc
 	golang.org/x/sync v0.7.0
-	golang.org/x/term v0.20.0
+	golang.org/x/term v0.22.0
 	gopkg.in/yaml.v3 v3.0.1
 	gotest.tools/v3 v3.5.1
 	sigs.k8s.io/yaml v1.4.0
@@ -108,10 +108,10 @@ require (
 	github.com/yuin/goldmark v1.5.4 // indirect
 	github.com/yuin/goldmark-emoji v1.0.2 // indirect
 	go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
-	golang.org/x/mod v0.17.0 // indirect
-	golang.org/x/net v0.24.0 // indirect
-	golang.org/x/sys v0.20.0 // indirect
-	golang.org/x/text v0.15.0 // indirect
-	golang.org/x/tools v0.20.0 // indirect
+	golang.org/x/mod v0.19.0 // indirect
+	golang.org/x/net v0.27.0 // indirect
+	golang.org/x/sys v0.22.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/tools v0.23.0 // indirect
 	mvdan.cc/gofumpt v0.6.0 // indirect
 )

go.sum

@@ -396,8 +396,8 @@ golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
-golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8=
+golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -419,8 +419,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
-golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
-golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
+golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
+golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -474,8 +474,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
-golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
+golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -485,8 +485,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
-golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
-golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
+golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk=
+golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -498,8 +498,8 @@ golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
-golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -531,8 +531,8 @@ golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
 golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM=
-golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY=
-golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg=
+golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg=
+golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

integration/scripts/cred_scopes.gpt

@@ -149,8 +149,8 @@ name: getcred
 import os
 import json
 
-var = os.getenv('var')
-val = os.getenv('val')
+var = os.getenv('VAR')
+val = os.getenv('VAL')
 
 output = {
     "env": {

pkg/engine/cmd.go

@@ -10,6 +10,7 @@ import (
 	"io"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"runtime"
 	"sort"
@@ -203,10 +204,8 @@ var ignoreENV = map[string]struct{}{
 }
 
 func appendEnv(envs []string, k, v string) []string {
-	for _, k := range []string{k, env.ToEnvLike(k)} {
-		if _, ignore := ignoreENV[k]; !ignore {
-			envs = append(envs, k+"="+v)
-		}
+	if _, ignore := ignoreENV[k]; !ignore {
+		envs = append(envs, strings.ToUpper(env.ToEnvLike(k))+"="+v)
 	}
 	return envs
 }
@@ -269,6 +268,10 @@ func (e *Engine) newCommand(ctx context.Context, extraEnv []string, tool types.T
 		})
 	}
 
+	if runtime.GOOS == "windows" && (args[0] == "/bin/bash" || args[0] == "/bin/sh") {
+		args[0] = path.Base(args[0])
+	}
+
 	if runtime.GOOS == "windows" && (args[0] == "/usr/bin/env" || args[0] == "/bin/env") {
 		args = args[1:]
 	}
@@ -279,7 +282,7 @@ func (e *Engine) newCommand(ctx context.Context, extraEnv []string, tool types.T
 	)
 
 	if strings.TrimSpace(rest) != "" {
-		f, err := os.CreateTemp("", version.ProgramName+requiredFileExtensions[args[0]])
+		f, err := os.CreateTemp(env.Getenv("GPTSCRIPT_TMPDIR", envvars), version.ProgramName+requiredFileExtensions[args[0]])
 		if err != nil {
 			return nil, nil, err
 		}

pkg/env/env.go

@@ -26,6 +26,15 @@ func ToEnvLike(v string) string {
 	return strings.ToUpper(v)
 }
 
+func Getenv(key string, envs []string) string {
+	for i := len(envs) - 1; i >= 0; i-- {
+		if k, v, ok := strings.Cut(envs[i], "="); ok && k == key {
+			return v
+		}
+	}
+	return ""
+}
+
 func Matches(cmd []string, bin string) bool {
 	switch len(cmd) {
 	case 0:

pkg/parser/parser.go

@@ -16,7 +16,7 @@ import (
 var (
 	sepRegex       = regexp.MustCompile(`^\s*---+\s*$`)
 	strictSepRegex = regexp.MustCompile(`^---\n$`)
-	skipRegex      = regexp.MustCompile(`^![-\w]+\s*$`)
+	skipRegex      = regexp.MustCompile(`^![-.:\w]+\s*$`)
 )
 
 func normalize(key string) string {
@@ -308,6 +308,8 @@ func Parse(input io.Reader, opts ...Options) (Document, error) {
 		}
 	}
 
+	nodes = assignMetadata(nodes)
+
 	if !opt.AssignGlobals {
 		return Document{
 			Nodes: nodes,
@@ -359,6 +361,42 @@ func Parse(input io.Reader, opts ...Options) (Document, error) {
 	}, nil
 }
 
+func assignMetadata(nodes []Node) (result []Node) {
+	metadata := map[string]map[string]string{}
+	result = make([]Node, 0, len(nodes))
+	for _, node := range nodes {
+		if node.TextNode != nil {
+			body, ok := strings.CutPrefix(node.TextNode.Text, "!metadata:")
+			if ok {
+				line, rest, ok := strings.Cut(body, "\n")
+				if ok {
+					toolName, metaKey, ok := strings.Cut(strings.TrimSpace(line), ":")
+					if ok {
+						d, ok := metadata[toolName]
+						if !ok {
+							d = map[string]string{}
+							metadata[toolName] = d
+						}
+						d[metaKey] = strings.TrimSpace(rest)
+					}
+				}
+			}
+		}
+	}
+	if len(metadata) == 0 {
+		return nodes
+	}
+
+	for _, node := range nodes {
+		if node.ToolNode != nil {
+			node.ToolNode.Tool.MetaData = metadata[node.ToolNode.Tool.Name]
+		}
+		result = append(result, node)
+	}
+
+	return
+}
+
 func isGPTScriptHashBang(line string) bool {
 	if !strings.HasPrefix(line, "#!") {
 		return false

pkg/parser/parser_test.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/gptscript-ai/gptscript/pkg/types"
 	"github.com/hexops/autogold/v2"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -239,3 +240,31 @@ share output filters: shared
 		}},
 	}}).Equal(t, out)
 }
+
+func TestParseMetaData(t *testing.T) {
+	input := `
+name: first
+
+body
+---
+!metadata:first:package.json
+foo=base
+f
+
+---
+!metadata:first2:requirements.txt
+asdf2
+
+---
+!metadata:first:requirements.txt
+asdf
+`
+	tools, err := ParseTools(strings.NewReader(input))
+	require.NoError(t, err)
+
+	assert.Len(t, tools, 1)
+	autogold.Expect(map[string]string{
+		"package.json":     "foo=base\nf",
+		"requirements.txt": "asdf",
+	}).Equal(t, tools[0].MetaData)
+}

pkg/repos/download/extract.go

@@ -9,7 +9,9 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"path"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/mholt/archiver/v4"
@@ -60,6 +62,18 @@ func Extract(ctx context.Context, downloadURL, digest, targetDir string) error {
 		return err
 	}
 
+	bin := path.Base(parsedURL.Path)
+	if strings.HasSuffix(bin, ".exe") {
+		dst, err := os.Create(filepath.Join(targetDir, bin))
+		if err != nil {
+			return err
+		}
+		defer dst.Close()
+
+		_, err = io.Copy(dst, tmpFile)
+		return err
+	}
+
 	format, input, err := archiver.Identify(filepath.Base(parsedURL.Path), tmpFile)
 	if err != nil {
 		return err

pkg/repos/get.go

@@ -15,6 +15,7 @@ import (
 	"github.com/BurntSushi/locker"
 	"github.com/gptscript-ai/gptscript/pkg/config"
 	"github.com/gptscript-ai/gptscript/pkg/credentials"
+	"github.com/gptscript-ai/gptscript/pkg/hash"
 	"github.com/gptscript-ai/gptscript/pkg/loader/github"
 	"github.com/gptscript-ai/gptscript/pkg/repos/git"
 	"github.com/gptscript-ai/gptscript/pkg/repos/runtimes/golang"
@@ -25,8 +26,8 @@ const credentialHelpersRepo = "github.com/gptscript-ai/gptscript-credential-help
 
 type Runtime interface {
 	ID() string
-	Supports(cmd []string) bool
-	Setup(ctx context.Context, dataRoot, toolSource string, env []string) ([]string, error)
+	Supports(tool types.Tool, cmd []string) bool
+	Setup(ctx context.Context, tool types.Tool, dataRoot, toolSource string, env []string) ([]string, error)
 }
 
 type noopRuntime struct {
@@ -36,11 +37,11 @@ func (n noopRuntime) ID() string {
 	return "none"
 }
 
-func (n noopRuntime) Supports(_ []string) bool {
+func (n noopRuntime) Supports(_ types.Tool, _ []string) bool {
 	return false
 }
 
-func (n noopRuntime) Setup(_ context.Context, _, _ string, _ []string) ([]string, error) {
+func (n noopRuntime) Setup(_ context.Context, _ types.Tool, _, _ string, _ []string) ([]string, error) {
 	return nil, nil
 }
 
@@ -200,11 +201,17 @@ func (m *Manager) setup(ctx context.Context, runtime Runtime, tool types.Tool, e
 	_ = os.RemoveAll(doneFile)
 	_ = os.RemoveAll(target)
 
-	if err := git.Checkout(ctx, m.gitDir, tool.Source.Repo.Root, tool.Source.Repo.Revision, target); err != nil {
-		return "", nil, err
+	if tool.Source.Repo.VCS == "git" {
+		if err := git.Checkout(ctx, m.gitDir, tool.Source.Repo.Root, tool.Source.Repo.Revision, target); err != nil {
+			return "", nil, err
+		}
+	} else {
+		if err := os.MkdirAll(target, 0755); err != nil {
+			return "", nil, err
+		}
 	}
 
-	newEnv, err := runtime.Setup(ctx, m.runtimeDir, targetFinal, env)
+	newEnv, err := runtime.Setup(ctx, tool, m.runtimeDir, targetFinal, env)
 	if err != nil {
 		return "", nil, err
 	}
@@ -227,20 +234,30 @@ func (m *Manager) setup(ctx context.Context, runtime Runtime, tool types.Tool, e
 }
 
 func (m *Manager) GetContext(ctx context.Context, tool types.Tool, cmd, env []string) (string, []string, error) {
+	var isLocal bool
 	if tool.Source.Repo == nil {
-		return tool.WorkingDir, env, nil
-	}
-
-	if tool.Source.Repo.VCS != "git" {
-		return "", nil, fmt.Errorf("only git is supported, found VCS %s for %s", tool.Source.Repo.VCS, tool.ID)
+		isLocal = true
+		d, _ := json.Marshal(tool)
+		id := hash.Digest(d)[:12]
+		tool.Source.Repo = &types.Repo{
+			VCS:      "<local>",
+			Root:     id,
+			Path:     "/",
+			Name:     id,
+			Revision: id,
+		}
 	}
 
 	for _, runtime := range m.runtimes {
-		if runtime.Supports(cmd) {
+		if runtime.Supports(tool, cmd) {
 			log.Debugf("Runtime %s supports %v", runtime.ID(), cmd)
 			return m.setup(ctx, runtime, tool, env)
 		}
 	}
 
+	if isLocal {
+		return tool.WorkingDir, env, nil
+	}
+
 	return m.setup(ctx, &noopRuntime{}, tool, env)
 }

pkg/repos/runtimes/busybox/SHASUMS256.txt

@@ -0,0 +1 @@
+6d2dfd1c1412c3550a89071a1b36a6f6073844320e687343d1dfc72719ecb8d9  FRP-5301-gda71f7c57/busybox-w64-FRP-5301-gda71f7c57.exe