diff --git a/docs/guide/conditional.md b/docs/guide/conditional.md index 0f6da9c..0425950 100644 --- a/docs/guide/conditional.md +++ b/docs/guide/conditional.md @@ -5,7 +5,7 @@ using a simple expression language that provides access to system information, e ### Expression Language -Flow uses the [Expr](https://expr-lang.org) language for evaluating conditions. The language supports common +flow uses the [Expr](https://expr-lang.org) language for evaluating conditions. The language supports common operators and functions while providing access to flow executable-specific context data. **See the [Expr language documentation](https://expr-lang.org/docs/language-definition) for more information on the diff --git a/docs/guide/templating.md b/docs/guide/templating.md index 885e8d3..fb4e0ad 100644 --- a/docs/guide/templating.md +++ b/docs/guide/templating.md @@ -100,18 +100,18 @@ files will only be copied if the `Type` field is set to `Helm`. The `deploy.sh` artifacts: - srcName: "helm-deploy.sh" srcDir: "scripts" # By default, the file will be copied from the template directory. This field can be used to specify a different directory. - if: "{{ eq .Type 'Helm' }}" # This will only copy the file if the Helm field is true + if: form["Type"] == "Helm" # This will only copy the file if the Helm field is true - srcName: "deploy.sh" srcDir: "scripts" - if: "{{ eq .Type 'K8s' }}" # This will only copy the file if the K8s field is true + if: form["Type"] == "K8s" # This will only copy the file if the K8s field is true - srcName: "values.yaml.tmpl" asTemplate: true dstName: "values.yaml" - if: "{{ eq .Type 'Helm' }}" # This will only copy the file if the Helm field is true + if: form["Type"] == "Helm" # This will only copy the file if the Helm field is true - srcName: "resources.yaml.tmpl" asTemplate: true dstName: "resources.yaml" - if: "{{ eq .Type 'K8s' }}" # This will only copy the file if the K8s field is true + if: form["Type"] == "K8s" # This will only copy the file if the K8s field is true ``` ### flowfile template string @@ -126,22 +126,22 @@ template: | tags: [k8s] executables: - verb: deploy - name: "{{ .FlowFileName }}" + name: "{{ name }}" exec: - file: "{{ if eq .Type 'Helm' }}helm-deploy.sh{{ else }}deploy.sh{{ end }}" + file: "{{ if form["Type"] == 'Helm' }}helm-deploy.sh{{ else }}deploy.sh{{ end }}" params: - envKey: "NAMESPACE" - text: "{{ .Namespace }}" + text: "{{ form["Namespace"] }}" - envKey: "APP_NAME" - text: "{{ .FlowFileName }}" + text: "{{ name }}" - verb: restart - name: "{{ .FlowFileName }}" + name: "{{ name }}" exec: - cmd: "kubectl rollout restart deployment/{{ .FlowFileName }} -n {{ .Namespace }}" + cmd: "kubectl rollout restart deployment/{{ name }} -n {{ form["Namespace"] }}" - verb: open - name: "{{ .FlowFileName }}" + name: "{{ name }}" launch: - uri: "https://{{ .FlowFileName }}.my.haus" + uri: "https://{{ name }}.my.haus" ``` ### Pre- and post- run executables @@ -156,14 +156,15 @@ Before exiting, it will also run a simple command and either open the flowfile i preRun: - ref: "validate k8s/validation:context" # You can reference other executables that you have on your system args: ["homelab"] - if: "{{ .Deploy }}" + if: form["Deploy"] postRun: - - cmd: "echo 'Rendered {{ if .Helm }}Helm values{{ else }}k8s manifest{{end}}'; ls -al" + - cmd: | + echo 'Rendered {{ if form["Helm"] }}Helm values{{ else }}k8s manifest{{end}}'; ls -al - ref: "edit vscode" - args: ["{{ .FlowFilePath }}"] - if: "{{ not .Deploy }}" - - ref: "deploy {{ .FlowFileName }}" - if: "{{ .Deploy }}" + args: ["{{ flowFilePath }}"] + if: not form["Deploy"] + - ref: "deploy {{ name }}" + if: form["Deploy"] ``` **Note**: preRun executables are run from the template directory, while postRun executables are run from the output directory. @@ -215,36 +216,56 @@ postRun: - ref: "edit vscode" args: ["{{ .FlowFilePath }}"] if: "{{ not .Deploy }}" - - ref: "deploy {{ .FlowFileName }}" + - ref: "deploy {{ .name }}" if: "{{ .Deploy }}" template: | tags: [k8s] executables: - verb: deploy - name: "{{ .FlowFileName }}" + name: "{{ name }}" exec: file: "{{ if eq .Type 'Helm' }}helm-deploy.sh{{ else }}deploy.sh{{ end }}" params: - envKey: "NAMESPACE" - text: "{{ .Namespace }}" + text: "{{ .form.namespace }}" - envKey: "APP_NAME" - text: "{{ .FlowFileName }}" + text: "{{ name }}" - verb: restart - name: "{{ .FlowFileName }}" + name: "{{ name }}" exec: cmd: "kubectl rollout restart deployment/{{ .FlowFileName }} -n {{ .Namespace }}" - verb: open - name: "{{ .FlowFileName }}" + name: "{{ name }}" launch: - uri: "https://{{ .FlowFileName }}.my.haus" -``` + uri: "https://{{ name }}.my.haus" +``` +### Templating language + +flow uses a hybrid of [Go text templating](https://pkg.go.dev/text/template) and [Expr](https://expr-lang.org) language for +rendering templates. + +Aside from the `if` field in the artifacts, preRun, and postRun sections, all other fields that use templating +will need to include the `{{` and `}}` delimiters to indicate that the text should be rendered as a template. + +**Template Variables** + +The following variables are automatically available in all template expressions: -### Template helpers +| Variable | Type | Description | +|-----------------|------------------------|------------------------------------------------------------------| +| `os` | string | Operating system identifier (e.g., "linux", "darwin", "windows") | +| `arch` | string | System architecture (e.g., "amd64", "arm64") | +| `workspace` | string | Target workspace name | +| `workspacePath` | string | Full path to the target workspace root directory | +| `name` | string | Name provided for the newly rendered flow file | +| `directory` | string | Target directory that the template will be render in to | +| `flowFilePath` | string | Full path to the target flow file | +| `templatePath` | string | Path to the template file being rendered | +| `env` | map (string -> string) | Environment variables accessible to the template | +| `form` | map (string -> any) | Values provided through template form inputs | -Templates can use the [Sprig functions](https://masterminds.github.io/sprig/) to manipulate data during the rendering process. -Additionally, the following keys are available to the template: +**See the [Expr language documentation](https://expr-lang.org/docs/language-definition) for more information on the +additional expression functions and syntax.** -- `FlowFileName`: The name of the flowfile being rendered. This is the argument passed into the `generate` command. -- `FlowFilePath`: The output path to the flowfile being rendered. -- `FlowWorkspace`: The name of the workspace that the template is rendering into. -- `FlowWorkspacePath`: The path to the workspace root directory. +> [!NOTE] +> The `env` map contains environment variables that were present when the template was rendered. The `form` map contains values from any form inputs defined in the template configuration. diff --git a/docs/schemas/template_schema.json b/docs/schemas/template_schema.json index 8ed2c28..f92fab3 100644 --- a/docs/schemas/template_schema.json +++ b/docs/schemas/template_schema.json @@ -30,7 +30,7 @@ "default": "" }, "if": { - "description": "A condition to determine if the artifact should be copied. The condition is evaluated using Go templating \nfrom the form data. If the condition is not met, the artifact will not be copied.\n[Sprig functions](https://masterminds.github.io/sprig/) are available for use in the condition.\n\nFor example, to copy the artifact only if the `name` field is set:\n```\n{{ if .name }}true{{ end }}\n```\n", + "description": "An expression that determines whether the the artifact should be copied, using the Expr language syntax. \nThe expression is evaluated at runtime and must resolve to a boolean value. If the condition is not met, \nthe artifact will not be copied.\n\nThe expression has access to OS/architecture information (os, arch), environment variables (env), form input \n(form), and context information (name, workspace, directory, etc.).\n\nSee the [flow documentation](https://flowexec.io/#/guide/templating) for more information.\n", "type": "string", "default": "" }, @@ -121,7 +121,7 @@ "default": "" }, "if": { - "description": "A condition to determine if the executable should be run. The condition is evaluated using Go templating \nfrom the form data. If the condition is not met, the executable run will be skipped.\n[Sprig functions](https://masterminds.github.io/sprig/) are available for use in the condition.\n\nFor example, to run a command only if the `name` field is set:\n```\n{{ if .name }}true{{ end }}\n```\n", + "description": "An expression that determines whether the executable should be run, using the Expr language syntax. \nThe expression is evaluated at runtime and must resolve to a boolean value. If the condition is not met, \nthe executable will be skipped.\n\nThe expression has access to OS/architecture information (os, arch), environment variables (env), form input \n(form), and context information (name, workspace, directory, etc.).\n\nSee the [flow documentation](https://flowexec.io/#/guide/templating) for more information.\n", "type": "string", "default": "" }, diff --git a/docs/types/template.md b/docs/types/template.md index b1d64f2..133bf4c 100644 --- a/docs/types/template.md +++ b/docs/types/template.md @@ -39,7 +39,7 @@ Go templating from form data is supported in all fields. | `asTemplate` | If true, the artifact will be copied as a template file. The file will be rendered using Go templating from the form data. [Sprig functions](https://masterminds.github.io/sprig/) are available for use in the template. | `boolean` | false | | | `dstDir` | The directory to copy the file to. If not set, the file will be copied to the root of the flow file directory. The directory will be created if it does not exist. | `string` | | | | `dstName` | The name of the file to copy to. If not set, the file will be copied with the same name. | `string` | | | -| `if` | A condition to determine if the artifact should be copied. The condition is evaluated using Go templating from the form data. If the condition is not met, the artifact will not be copied. [Sprig functions](https://masterminds.github.io/sprig/) are available for use in the condition. For example, to copy the artifact only if the `name` field is set: ``` {{ if .name }}true{{ end }} ``` | `string` | | | +| `if` | An expression that determines whether the the artifact should be copied, using the Expr language syntax. The expression is evaluated at runtime and must resolve to a boolean value. If the condition is not met, the artifact will not be copied. The expression has access to OS/architecture information (os, arch), environment variables (env), form input (form), and context information (name, workspace, directory, etc.). See the [flow documentation](https://flowexec.io/#/guide/templating) for more information. | `string` | | | | `srcDir` | The directory to copy the file from. If not set, the file will be copied from the directory of the template file. | `string` | | | | `srcName` | The name of the file to copy. | `string` | | | @@ -94,7 +94,7 @@ Configuration for a template executable. | ----- | ----------- | ---- | ------- | :--------: | | `args` | Arguments to pass to the executable. | `array` (`string`) | [] | | | `cmd` | The command to execute. One of `cmd` or `ref` must be set. | `string` | | | -| `if` | A condition to determine if the executable should be run. The condition is evaluated using Go templating from the form data. If the condition is not met, the executable run will be skipped. [Sprig functions](https://masterminds.github.io/sprig/) are available for use in the condition. For example, to run a command only if the `name` field is set: ``` {{ if .name }}true{{ end }} ``` | `string` | | | +| `if` | An expression that determines whether the executable should be run, using the Expr language syntax. The expression is evaluated at runtime and must resolve to a boolean value. If the condition is not met, the executable will be skipped. The expression has access to OS/architecture information (os, arch), environment variables (env), form input (form), and context information (name, workspace, directory, etc.). See the [flow documentation](https://flowexec.io/#/guide/templating) for more information. | `string` | | | | `ref` | A reference to another executable to run in serial. One of `cmd` or `ref` must be set. | [ExecutableRef](#ExecutableRef) | | | diff --git a/examples/exec-template.flow.tmpl b/examples/exec-template.flow.tmpl index a595d2d..689127a 100644 --- a/examples/exec-template.flow.tmpl +++ b/examples/exec-template.flow.tmpl @@ -10,7 +10,7 @@ template: | namespace: examples executables: - verb: run - name: "{{ .Color }}-msg" - description: Created from a template in {{ .FlowWorkspace }} + name: {{ form["Color"] }}-msg + description: Created from a template in {{ workspace }} exec: cmd: cat message.txt diff --git a/internal/services/expr/expr.go b/internal/services/expr/expr.go index 69aea8b..7892b76 100644 --- a/internal/services/expr/expr.go +++ b/internal/services/expr/expr.go @@ -13,7 +13,7 @@ import ( "github.com/jahvon/flow/types/executable" ) -func IsTruthy(ex string, env *ExpressionData) (bool, error) { +func IsTruthy(ex string, env any) (bool, error) { output, err := Evaluate(ex, env) if err != nil { return false, err @@ -35,7 +35,7 @@ func IsTruthy(ex string, env *ExpressionData) (bool, error) { } } -func Evaluate(ex string, env *ExpressionData) (interface{}, error) { +func Evaluate(ex string, env any) (interface{}, error) { program, err := expr.Compile(ex, expr.Env(env)) if err != nil { return nil, err @@ -48,7 +48,7 @@ func Evaluate(ex string, env *ExpressionData) (interface{}, error) { return output, nil } -func EvaluateString(ex string, env *ExpressionData) (string, error) { +func EvaluateString(ex string, env any) (string, error) { output, err := Evaluate(ex, env) if err != nil { return "", err @@ -64,6 +64,7 @@ type CtxData struct { Workspace string `expr:"workspace"` Namespace string `expr:"namespace"` WorkspacePath string `expr:"workspacePath"` + FlowFileName string `expr:"flowFileName"` FlowFilePath string `expr:"flowFilePath"` FlowFileDir string `expr:"flowFileDir"` } @@ -81,13 +82,15 @@ func ExpressionEnv( executable *executable.Executable, dataMap, envMap map[string]string, ) ExpressionData { + fn := filepath.Base(filepath.Base(executable.FlowFilePath())) return ExpressionData{ OS: runtime.GOOS, Arch: runtime.GOARCH, Ctx: &CtxData{ Workspace: ctx.CurrentWorkspace.AssignedName(), - Namespace: ctx.CurrentWorkspace.AssignedName(), + Namespace: ctx.Config.CurrentNamespace, WorkspacePath: executable.WorkspacePath(), + FlowFileName: fn, FlowFilePath: executable.FlowFilePath(), FlowFileDir: filepath.Dir(executable.FlowFilePath()), }, diff --git a/internal/services/expr/expr_test.go b/internal/services/expr/expr_test.go index 74773d5..0a88e44 100644 --- a/internal/services/expr/expr_test.go +++ b/internal/services/expr/expr_test.go @@ -80,7 +80,7 @@ var _ = Describe("Expr", func() { }) }) - var _ = Describe("ExpressionData", func() { + Describe("ExpressionData", func() { var ( data *expr.ExpressionData ) diff --git a/internal/services/expr/template.go b/internal/services/expr/template.go new file mode 100644 index 0000000..fff3d26 --- /dev/null +++ b/internal/services/expr/template.go @@ -0,0 +1,186 @@ +package expr + +import ( + "bytes" + "fmt" + "io" + "strconv" + "strings" + "text/template" + + "github.com/expr-lang/expr" + "github.com/expr-lang/expr/vm" +) + +// Template wraps text/template but evaluates expressions using expr instead +type Template struct { + name string + text string + data any + tmpl *template.Template + exprCache map[string]*vm.Program +} + +func NewTemplate(name string, data any) *Template { + t := &Template{ + name: name, + data: data, + exprCache: make(map[string]*vm.Program), + } + return t +} + +func (t *Template) Parse(text string) error { + t.text = text + processed := t.preProcessExpressions(text) + tmpl := template.New(t.name).Funcs(template.FuncMap{"expr": t.evalExpr, "exprBool": t.evalExprBool}) + + parsed, err := tmpl.Parse(processed) + if err != nil { + return fmt.Errorf("parsing template: %w", err) + } + + t.tmpl = parsed + return nil +} + +func (t *Template) Execute(wr io.Writer) error { + if t.tmpl == nil { + return fmt.Errorf("template not parsed") + } + + return t.tmpl.Execute(wr, t.data) +} + +func (t *Template) ExecuteToString() (string, error) { + var buf bytes.Buffer + err := t.Execute(&buf) + return buf.String(), err +} + +func (t *Template) compileExpr(expression string) (*vm.Program, error) { + if node, ok := t.exprCache[expression]; ok { + return node, nil + } + + compiled, err := expr.Compile(expression, expr.Env(t.data)) + if err != nil { + return nil, err + } + t.exprCache[expression] = compiled + return compiled, nil +} + +//nolint:funlen +func (t *Template) preProcessExpressions(text string) string { + var result strings.Builder + remaining := text + contextDepth := 0 // Track nested range/with blocks + + for { + start := strings.Index(remaining, "{{") + if start == -1 { + result.WriteString(remaining) + break + } + result.WriteString(remaining[:start]) + + end := strings.Index(remaining[start:], "}}") + if end == -1 { + result.WriteString(remaining[start:]) + break + } + end += start + + action := remaining[start+2 : end] + trimLeft := strings.HasPrefix(action, "-") + trimRight := strings.HasSuffix(action, "-") + action = strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(action, "-"), "-")) + + result.WriteString("{{") + if trimLeft { + result.WriteString("-") + } + result.WriteString(" ") + + switch { + case strings.HasPrefix(action, "if "): + condition := strings.TrimPrefix(action, "if ") + result.WriteString("if exprBool `") + result.WriteString(strings.TrimSpace(condition)) + result.WriteString("`") + case strings.HasPrefix(action, "with "): + value := strings.TrimPrefix(action, "with ") + result.WriteString("with expr `") + result.WriteString(strings.TrimSpace(value)) + result.WriteString("`") + contextDepth++ + case action == "end": + result.WriteString("end") + if contextDepth > 0 { + contextDepth-- + } + case action == "else": + result.WriteString("else") + case strings.HasPrefix(action, "range "): + value := strings.TrimPrefix(action, "range ") + result.WriteString("range expr `") + result.WriteString(strings.TrimSpace(value)) + result.WriteString("`") + contextDepth++ + default: + if contextDepth > 0 && (strings.HasPrefix(action, ".") || action == ".") { + result.WriteString(action) + } else { + result.WriteString("expr `") + result.WriteString(strings.TrimSpace(action)) + result.WriteString("`") + } + } + + result.WriteString(" ") + if trimRight { + result.WriteString("-") + } + result.WriteString("}}") + + remaining = remaining[end+2:] + } + + return result.String() +} + +func (t *Template) evalExpr(expression string) (interface{}, error) { + program, err := t.compileExpr(expression) + if err != nil { + return nil, fmt.Errorf("compiling expression: %w", err) + } + result, err := expr.Run(program, t.data) + if err != nil { + return nil, fmt.Errorf("evaluating expression: %w", err) + } + + return result, nil +} + +func (t *Template) evalExprBool(expression string) (bool, error) { + result, err := t.evalExpr(expression) + if err != nil { + return false, err + } + + switch v := result.(type) { + case bool: + return v, nil + case int, int64, float64, uint, uint64: + return v != 0, nil + case string: + truthy, err := strconv.ParseBool(strings.Trim(v, `"' `)) + if err != nil { + return false, err + } + return truthy, nil + default: + return result != nil, nil + } +} diff --git a/internal/services/expr/template_test.go b/internal/services/expr/template_test.go new file mode 100644 index 0000000..ed78c39 --- /dev/null +++ b/internal/services/expr/template_test.go @@ -0,0 +1,183 @@ +package expr_test + +import ( + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/jahvon/flow/internal/services/expr" +) + +var _ = Describe("Template", func() { + var ( + tmpl *expr.Template + data map[string]interface{} + ) + + BeforeEach(func() { + data = map[string]interface{}{ + "os": "linux", + "arch": "amd64", + "store": map[string]interface{}{"key1": "value1", "key2": 2}, + "ctx": map[string]interface{}{"workspace": "test_workspace", "namespace": "test_namespace"}, + "workspaces": []string{"test_workspace", "other_workspace"}, + "executables": []map[string]interface{}{ + {"name": "exec1", "tags": []string{"tag"}, "type": "serial"}, + {"name": "exec2", "tags": []string{}, "type": "exec"}, + {"name": "exec3", "tags": []string{"tag", "tag2"}, "type": "exec"}, + }, + "featureEnabled": true, + } + tmpl = expr.NewTemplate("test", data) + }) + + Describe("expr evaluation", func() { + It("evaluates simple expressions", func() { + err := tmpl.Parse("{{ ctx.workspace }}") + Expect(err).NotTo(HaveOccurred()) + + result, err := tmpl.ExecuteToString() + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal("test_workspace")) + }) + + It("evaluates boolean expressions", func() { + err := tmpl.Parse("{{ os == \"linux\" && arch == \"amd64\" }}") + Expect(err).NotTo(HaveOccurred()) + + result, err := tmpl.ExecuteToString() + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal("true")) + }) + + It("evaluates arithmetic expressions", func() { + err := tmpl.Parse("{{ store[\"key2\"] * 2 }}") + Expect(err).NotTo(HaveOccurred()) + + result, err := tmpl.ExecuteToString() + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal("4")) + }) + }) + + Describe("control structures", func() { + It("handles if/else with expr conditions", func() { + template := ` + {{- if featureEnabled && ctx.workspace == "test_workspace" }} + Matched + {{- else }} + Unmatched + {{- end }} + ` + err := tmpl.Parse(template) + Expect(err).NotTo(HaveOccurred()) + + result, err := tmpl.ExecuteToString() + Expect(err).NotTo(HaveOccurred()) + Expect(strings.TrimSpace(result)).To(Equal("Matched")) + }) + + It("handles range with expr", func() { + template := ` +{{- range filter(executables, {.type == "exec"}) }} +{{ .name }}: {{ .tags }} +{{- end }} + ` + err := tmpl.Parse(template) + Expect(err).NotTo(HaveOccurred()) + + result, err := tmpl.ExecuteToString() + Expect(err).NotTo(HaveOccurred()) + expected := "exec2: []\nexec3: [tag tag2]" + Expect(strings.TrimSpace(result)).To(Equal(expected)) + }) + + It("handles with using expr", func() { + template := ` +{{- with ctx }} +Workspace: {{ .workspace }} +Namespace: {{ .namespace }} +{{- end }} + ` + err := tmpl.Parse(template) + Expect(err).NotTo(HaveOccurred()) + + result, err := tmpl.ExecuteToString() + Expect(err).NotTo(HaveOccurred()) + expected := "Workspace: test_workspace\nNamespace: test_namespace" + Expect(strings.TrimSpace(result)).To(Equal(expected)) + }) + + It("handles nested control structures with expr", func() { + GinkgoT().Skip("nested control structures not supported yet") + template := ` +{{- range executables }} +{{- $exec := . }} +{{- if len($exec.tags) > 0 }} +{{ .name }}: {{ .type }} +{{- end }} +{{- end }} + ` + err := tmpl.Parse(template) + Expect(err).NotTo(HaveOccurred()) + + result, err := tmpl.ExecuteToString() + Expect(err).NotTo(HaveOccurred()) + expected := "Item 1: 12.089 (with tax)\nItem 3: 16.5 (with tax)" + Expect(strings.TrimSpace(result)).To(Equal(expected)) + }) + }) + + Describe("error handling", func() { + It("handles invalid expressions", func() { + err := tmpl.Parse("{{ unknown.field }}") + Expect(err).NotTo(HaveOccurred()) + + _, err = tmpl.ExecuteToString() + Expect(err).To(HaveOccurred()) + }) + + It("handles invalid syntax in if conditions", func() { + err := tmpl.Parse("{{ if 1 ++ \"2\" }}invalid{{end}}") + Expect(err).NotTo(HaveOccurred()) + + _, err = tmpl.ExecuteToString() + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("Template with trim markers", func() { + It("handles trim markers in range", func() { + template := `start +{{- range workspaces }} +{{ . }} +{{- end }} +end` + err := tmpl.Parse(template) + Expect(err).NotTo(HaveOccurred()) + + result, err := tmpl.ExecuteToString() + Expect(err).NotTo(HaveOccurred()) + expected := "start\ntest_workspace\nother_workspace\nend" + Expect(result).To(Equal(expected)) + }) + + It("handles trim markers in if/else", func() { + template := `start +{{- if featureEnabled }} +enabled +{{- else }} +disabled +{{- end }} +end` + err := tmpl.Parse(template) + Expect(err).NotTo(HaveOccurred()) + + result, err := tmpl.ExecuteToString() + Expect(err).NotTo(HaveOccurred()) + expected := "start\nenabled\nend" + Expect(result).To(Equal(expected)) + }) + }) +}) diff --git a/internal/templates/artifacts.go b/internal/templates/artifacts.go index 7c7d3fd..6c7040c 100644 --- a/internal/templates/artifacts.go +++ b/internal/templates/artifacts.go @@ -11,6 +11,7 @@ import ( "github.com/pkg/errors" "github.com/jahvon/flow/internal/filesystem" + "github.com/jahvon/flow/internal/services/expr" "github.com/jahvon/flow/types/executable" ) @@ -18,12 +19,12 @@ func copyAllArtifacts( logger tuikitIO.Logger, artifacts []executable.Artifact, wsDir, srcDir, dstDir string, - data, envMap map[string]string, + templateData expressionData, ) error { var errs []error for i, a := range artifacts { if err := copyArtifact( - logger, fmt.Sprintf("artifact-%d", i), wsDir, srcDir, dstDir, a, data, envMap, + logger, fmt.Sprintf("artifact-%d", i), wsDir, srcDir, dstDir, a, templateData, ); err != nil { errs = append(errs, err) } @@ -39,15 +40,15 @@ func copyArtifact( logger tuikitIO.Logger, name, wsPath, srcDir, dstDir string, artifact executable.Artifact, - data, envMap map[string]string, + templateData expressionData, ) error { - srcPath, err := parseSourcePath(logger, name, srcDir, wsPath, artifact, data, envMap) + srcPath, err := parseSourcePath(logger, name, srcDir, wsPath, artifact, templateData) if err != nil { return errors.Wrap(err, "unable to parse source path") } if artifact.If != "" { - eval, err := goTemplateEvaluatedTrue(name, artifact.If, data) + eval, err := expr.IsTruthy(artifact.If, templateData) if err != nil { return errors.Wrap(err, "unable to evaluate if condition") } @@ -68,7 +69,7 @@ func copyArtifact( m := artifact m.SrcName = filepath.Base(match) m.SrcDir = filepath.Dir(match) - mErr := copyArtifact(logger, fmt.Sprintf("%s-%d", name, i), wsPath, srcDir, dstDir, m, data, envMap) + mErr := copyArtifact(logger, fmt.Sprintf("%s-%d", name, i), wsPath, srcDir, dstDir, m, templateData) if mErr != nil { errs = append(errs, mErr) } @@ -96,7 +97,7 @@ func copyArtifact( a.SrcName = filepath.Base(path) a.SrcDir = filepath.Dir(path) aName := fmt.Sprintf("%s-%s", name, a.SrcName) - return copyArtifact(logger, aName, wsPath, srcDir, dstDir, a, data, envMap) + return copyArtifact(logger, aName, wsPath, srcDir, dstDir, a, templateData) }) if err != nil { return errors.Wrap(err, "unable to walk directory") @@ -110,7 +111,7 @@ func copyArtifact( name, dstDir, srcDir, wsPath, artifact, - data, envMap, + templateData, ) if err != nil { return errors.Wrap(err, "unable to parse destination path") diff --git a/internal/templates/expression.go b/internal/templates/expression.go new file mode 100644 index 0000000..642d9bf --- /dev/null +++ b/internal/templates/expression.go @@ -0,0 +1,30 @@ +package templates + +import "runtime" + +type expressionData map[string]interface{} + +func newExpressionData( + ws, wsPath, flowfileName, flowfileDir, flowfilePath, templatePath string, + envMap, formMap map[string]string, +) expressionData { + return map[string]interface{}{ + "os": runtime.GOOS, + "arch": runtime.GOARCH, + "workspace": ws, + "workspacePath": wsPath, + "name": flowfileName, + "directory": flowfileDir, + "flowFilePath": flowfilePath, + "templatePath": templatePath, + "env": envMap, + "form": formMap, + } +} + +func expressionEnv(data expressionData) map[string]string { + if env, ok := data["env"].(map[string]string); ok { + return env + } + return nil +} diff --git a/internal/templates/templates.go b/internal/templates/templates.go index e844681..5f1ac92 100644 --- a/internal/templates/templates.go +++ b/internal/templates/templates.go @@ -6,12 +6,9 @@ import ( "maps" "os" "path/filepath" - "strconv" "strings" - "text/template" "time" - "github.com/Masterminds/sprig/v3" tuikitIO "github.com/jahvon/tuikit/io" "github.com/pkg/errors" "gopkg.in/yaml.v3" @@ -20,6 +17,7 @@ import ( "github.com/jahvon/flow/internal/filesystem" "github.com/jahvon/flow/internal/runner" "github.com/jahvon/flow/internal/runner/engine" + "github.com/jahvon/flow/internal/services/expr" "github.com/jahvon/flow/internal/utils" argUtils "github.com/jahvon/flow/internal/utils/args" execUtils "github.com/jahvon/flow/internal/utils/executables" @@ -42,12 +40,12 @@ func ProcessTemplate( flowfileName += executable.FlowFileExt } - data := make(map[string]string) + formMap := make(map[string]string) if template.Form != nil { if err := showForm(ctx, template.Form); err != nil { return err } - data = template.Form.ValueMap() + formMap = template.Form.ValueMap() } env := os.Environ() @@ -63,13 +61,14 @@ func ProcessTemplate( "template", template.Location(), "output", fullPath, ) - data["FlowWorkspace"] = ws.AssignedName() - data["FlowWorkspacePath"] = ws.Location() - data["FlowFileName"] = flowfileName - data["FlowFilePath"] = fullPath + dataMap := newExpressionData( + ws.AssignedName(), ws.Location(), + flowfileName, flowfileDir, fullPath, template.Location(), + envMap, formMap, + ) if err := runExecutables( - ctx, ws, "pre-run", filepath.Dir(template.Location()), template.PreRun, envMap, data, + ctx, ws, "pre-run", filepath.Dir(template.Location()), template.PreRun, dataMap, ); err != nil { return err } @@ -79,13 +78,13 @@ func ProcessTemplate( ws.Location(), filepath.Dir(template.Location()), flowfileDir, - data, envMap, + dataMap, ); err != nil { return err } if template.Template != "" { - flowfile, err := templateToFlowfile(template, data) + flowfile, err := templateToFlowfile(template, dataMap) if err != nil { return err } @@ -99,7 +98,7 @@ func ProcessTemplate( return errors.Wrap(err, fmt.Sprintf("unable to write flowfile %s from template", flowfileName)) } } - if err := runExecutables(ctx, ws, "post-run", flowfileDir, template.PostRun, envMap, data); err != nil { + if err := runExecutables(ctx, ws, "post-run", flowfileDir, template.PostRun, dataMap); err != nil { return err } @@ -112,13 +111,12 @@ func runExecutables( ws *workspace.Workspace, stage, flowfileDir string, execs []executable.TemplateRefConfig, - envMap map[string]string, - templateData map[string]string, + templateData expressionData, ) error { ctx.Logger.Debugf("running %d %s executables", len(execs), stage) for i, e := range execs { if e.If != "" { - eval, err := goTemplateEvaluatedTrue(flowfileDir, e.If, templateData) + eval, err := expr.IsTruthy(e.If, templateData) if err != nil { return errors.Wrap(err, "unable to evaluate if condition") } @@ -149,7 +147,8 @@ func runExecutables( return errors.New("post-run executable must have a ref or cmd") } execEnv := make(map[string]string) - maps.Copy(execEnv, envMap) + ee := expressionEnv(templateData) + maps.Copy(execEnv, ee) if len(e.Args) > 0 { args := make([]string, 0) for _, arg := range e.Args { @@ -159,7 +158,7 @@ func runExecutables( } args = append(args, a.String()) } - a, err := argUtils.ProcessArgs(exec, args, envMap) + a, err := argUtils.ProcessArgs(exec, args, ee) if err != nil { ctx.Logger.Error(err, "unable to process arguments") } @@ -180,14 +179,14 @@ func parseSourcePath( logger tuikitIO.Logger, name, flowFileSrc, wsDir string, artifact executable.Artifact, - data, envMap map[string]string, + templateData expressionData, ) (string, error) { var err error if artifact.SrcDir != "" { - flowFileSrc = utils.ExpandDirectory(logger, artifact.SrcDir, wsDir, flowFileSrc, envMap) + flowFileSrc = utils.ExpandDirectory(logger, artifact.SrcDir, wsDir, flowFileSrc, expressionEnv(templateData)) } var sb *bytes.Buffer - sb, err = processAsGoTemplate(name, filepath.Join(flowFileSrc, artifact.SrcName), data) + sb, err = processAsGoTemplate(name, filepath.Join(flowFileSrc, artifact.SrcName), templateData) if err != nil { return "", errors.Wrap(err, "unable to process artifact as template") } @@ -198,15 +197,15 @@ func parseDestinationPath( logger tuikitIO.Logger, name, dstDir, flowFileSrc, wsDir string, artifact executable.Artifact, - data, envMap map[string]string, + templateData expressionData, ) (string, error) { var err error if artifact.DstDir != "" { - dstDir = utils.ExpandDirectory(logger, artifact.DstDir, wsDir, flowFileSrc, envMap) + dstDir = utils.ExpandDirectory(logger, artifact.DstDir, wsDir, flowFileSrc, expressionEnv(templateData)) } dstName := artifact.DstName var db *bytes.Buffer - db, err = processAsGoTemplate(name, dstName, data) + db, err = processAsGoTemplate(name, dstName, templateData) if err != nil { return "", errors.Wrap(err, "unable to process artifact as template") } @@ -216,9 +215,9 @@ func parseDestinationPath( func templateToFlowfile( t *executable.Template, - data map[string]string, + templateData expressionData, ) (*executable.FlowFile, error) { - buf, err := processAsGoTemplate(t.Name(), t.Template, data) + buf, err := processAsGoTemplate(t.Name(), t.Template, templateData) if err != nil { return nil, errors.Wrap(err, fmt.Sprintf("flowfile template %s", t.Name())) } @@ -231,32 +230,20 @@ func templateToFlowfile( return flowfile, nil } -func processAsGoTemplate(fileName, txt string, data map[string]string) (*bytes.Buffer, error) { - tmpl, err := template.New(fileName).Funcs(sprig.TxtFuncMap()).Parse(txt) - if err != nil { +func processAsGoTemplate(fileName, txt string, data expressionData) (*bytes.Buffer, error) { + tmpl := expr.NewTemplate(fileName, data) + if err := tmpl.Parse(txt); err != nil { return nil, errors.Wrap(err, fmt.Sprintf("unable to parse %s template", fileName)) } var buf bytes.Buffer - if err := tmpl.Execute(&buf, data); err != nil { + if err := tmpl.Execute(&buf); err != nil { return nil, errors.Wrap(err, fmt.Sprintf("unable to execute %s template", fileName)) } return &buf, nil } -func goTemplateEvaluatedTrue(fileName, txt string, data map[string]string) (bool, error) { - t, err := template.New(fileName).Funcs(sprig.FuncMap()).Parse(txt) - if err != nil { - return false, errors.Wrap(err, fmt.Sprintf("unable to parse %s template", fileName)) - } - var buf bytes.Buffer - if err := t.Execute(&buf, data); err != nil { - return false, errors.Wrap(err, "unable to evaluate template") - } - return strconv.ParseBool(buf.String()) -} - // templateParent returns a pseudo-executable that can be used as a parent for other executables. It simply includes // the executable context that is derived from the rendered template. func templateParent(ws, wsPath, flowfilePath string) *executable.Executable { diff --git a/tests/template_cmds_e2e_test.go b/tests/template_cmds_e2e_test.go index f95787b..77a2df4 100644 --- a/tests/template_cmds_e2e_test.go +++ b/tests/template_cmds_e2e_test.go @@ -34,8 +34,8 @@ var _ = Describe("flowfile template commands e2e", Ordered, func() { Executables: []*executable.Executable{ { Verb: "exec", - Name: "{{ .Name }}", - Exec: &executable.ExecExecutableType{Cmd: fmt.Sprintf("echo '%s'", "{{ .Msg }}")}}, + Name: "{{ name }}", + Exec: &executable.ExecExecutableType{Cmd: fmt.Sprintf("echo '%s'", "{{ form['Msg'] }}")}}, }, } tmplStr, err := tmpl.YAML() @@ -63,7 +63,7 @@ var _ = Describe("flowfile template commands e2e", Ordered, func() { }, PostRun: []executable.TemplateRefConfig{ { - Cmd: "touch {{ .Name }}", + Cmd: "touch {{ name }}", }, }, } diff --git a/types/executable/template.gen.go b/types/executable/template.gen.go index 1d1cfea..ba30150 100644 --- a/types/executable/template.gen.go +++ b/types/executable/template.gen.go @@ -22,17 +22,18 @@ type Artifact struct { // same name. DstName string `json:"dstName,omitempty" yaml:"dstName,omitempty" mapstructure:"dstName,omitempty"` - // A condition to determine if the artifact should be copied. The condition is - // evaluated using Go templating - // from the form data. If the condition is not met, the artifact will not be - // copied. - // [Sprig functions](https://masterminds.github.io/sprig/) are available for use - // in the condition. + // An expression that determines whether the the artifact should be copied, using + // the Expr language syntax. + // The expression is evaluated at runtime and must resolve to a boolean value. If + // the condition is not met, + // the artifact will not be copied. // - // For example, to copy the artifact only if the `name` field is set: - // ``` - // {{ if .name }}true{{ end }} - // ``` + // The expression has access to OS/architecture information (os, arch), + // environment variables (env), form input + // (form), and context information (name, workspace, directory, etc.). + // + // See the [flow documentation](https://flowexec.io/#/guide/templating) for more + // information. // If string `json:"if,omitempty" yaml:"if,omitempty" mapstructure:"if,omitempty"` @@ -127,17 +128,18 @@ type TemplateRefConfig struct { // Cmd string `json:"cmd,omitempty" yaml:"cmd,omitempty" mapstructure:"cmd,omitempty"` - // A condition to determine if the executable should be run. The condition is - // evaluated using Go templating - // from the form data. If the condition is not met, the executable run will be - // skipped. - // [Sprig functions](https://masterminds.github.io/sprig/) are available for use - // in the condition. + // An expression that determines whether the executable should be run, using the + // Expr language syntax. + // The expression is evaluated at runtime and must resolve to a boolean value. If + // the condition is not met, + // the executable will be skipped. + // + // The expression has access to OS/architecture information (os, arch), + // environment variables (env), form input + // (form), and context information (name, workspace, directory, etc.). // - // For example, to run a command only if the `name` field is set: - // ``` - // {{ if .name }}true{{ end }} - // ``` + // See the [flow documentation](https://flowexec.io/#/guide/templating) for more + // information. // If string `json:"if,omitempty" yaml:"if,omitempty" mapstructure:"if,omitempty"` diff --git a/types/executable/template_schema.yaml b/types/executable/template_schema.yaml index acf582c..579a006 100644 --- a/types/executable/template_schema.yaml +++ b/types/executable/template_schema.yaml @@ -15,15 +15,15 @@ definitions: properties: if: type: string - description: | - A condition to determine if the artifact should be copied. The condition is evaluated using Go templating - from the form data. If the condition is not met, the artifact will not be copied. - [Sprig functions](https://masterminds.github.io/sprig/) are available for use in the condition. + description: | + An expression that determines whether the the artifact should be copied, using the Expr language syntax. + The expression is evaluated at runtime and must resolve to a boolean value. If the condition is not met, + the artifact will not be copied. + + The expression has access to OS/architecture information (os, arch), environment variables (env), form input + (form), and context information (name, workspace, directory, etc.). - For example, to copy the artifact only if the `name` field is set: - ``` - {{ if .name }}true{{ end }} - ``` + See the [flow documentation](https://flowexec.io/#/guide/templating) for more information. default: "" asTemplate: type: boolean @@ -125,14 +125,14 @@ definitions: if: type: string description: | - A condition to determine if the executable should be run. The condition is evaluated using Go templating - from the form data. If the condition is not met, the executable run will be skipped. - [Sprig functions](https://masterminds.github.io/sprig/) are available for use in the condition. + An expression that determines whether the executable should be run, using the Expr language syntax. + The expression is evaluated at runtime and must resolve to a boolean value. If the condition is not met, + the executable will be skipped. - For example, to run a command only if the `name` field is set: - ``` - {{ if .name }}true{{ end }} - ``` + The expression has access to OS/architecture information (os, arch), environment variables (env), form input + (form), and context information (name, workspace, directory, etc.). + + See the [flow documentation](https://flowexec.io/#/guide/templating) for more information. default: "" type: object