From e1cd58bf16500669d0e15f0c2c4a2d8308922abb Mon Sep 17 00:00:00 2001 From: Levko Burburas Date: Tue, 26 Nov 2024 01:29:35 +0200 Subject: [PATCH 01/21] fix: wrap tf stdout and stderr in json --- cli/commands/flags.go | 14 +++------ cli/deprecated_flags.go | 23 +++++++++++++++ config/dependency.go | 1 - configstack/running_module.go | 1 - internal/strict/strict.go | 6 ++++ options/options.go | 5 ---- pkg/log/format/format.go | 7 +++-- pkg/log/format/placeholders/field.go | 12 ++++++-- shell/run_shell_cmd.go | 44 ++++++++++++++++------------ 9 files changed, 72 insertions(+), 41 deletions(-) diff --git a/cli/commands/flags.go b/cli/commands/flags.go index 4d959f264..095d8afb1 100644 --- a/cli/commands/flags.go +++ b/cli/commands/flags.go @@ -68,9 +68,6 @@ const ( TerragruntDebugFlagName = "terragrunt-debug" TerragruntDebugEnvName = "TERRAGRUNT_DEBUG" - TerragruntTfLogJSONFlagName = "terragrunt-tf-logs-to-json" - TerragruntTfLogJSONEnvName = "TERRAGRUNT_TF_JSON_LOG" - TerragruntModulesThatIncludeFlagName = "terragrunt-modules-that-include" TerragruntModulesThatIncludeEnvName = "TERRAGRUNT_MODULES_THAT_INCLUDE" @@ -401,12 +398,6 @@ func NewGlobalFlags(opts *options.TerragruntOptions) cli.Flags { return nil }, }, - &cli.BoolFlag{ - Name: TerragruntTfLogJSONFlagName, - EnvVar: TerragruntTfLogJSONEnvName, - Destination: &opts.TerraformLogsToJSON, - Usage: "If specified, Terragrunt will wrap Terraform stdout and stderr in JSON.", - }, &cli.BoolFlag{ Name: TerragruntUsePartialParseConfigCacheFlagName, EnvVar: TerragruntUsePartialParseConfigCacheEnvName, @@ -439,8 +430,11 @@ func NewGlobalFlags(opts *options.TerragruntOptions) cli.Flags { return nil } - if val == format.BareFormatName { + switch val { + case format.BareFormatName: opts.ForwardTFStdout = true + case format.JSONFormatName: + opts.JSONLogFormat = true } opts.LogFormatter.SetFormat(phs) diff --git a/cli/deprecated_flags.go b/cli/deprecated_flags.go index 8f2030683..52a9ba36d 100644 --- a/cli/deprecated_flags.go +++ b/cli/deprecated_flags.go @@ -19,6 +19,9 @@ const ( TerragruntJSONLogFlagName = "terragrunt-json-log" TerragruntJSONLogEnvName = "TERRAGRUNT_JSON_LOG" + + TerragruntTfLogJSONFlagName = "terragrunt-tf-logs-to-json" + TerragruntTfLogJSONEnvName = "TERRAGRUNT_TF_JSON_LOG" ) // NewDeprecatedFlags creates and returns deprecated flags. @@ -39,6 +42,7 @@ func NewDeprecatedFlags(opts *options.TerragruntOptions) cli.Flags { EnvVar: TerragruntDisableLogFormattingEnvName, Destination: &opts.DisableLogFormatting, Usage: "If specified, logs will be displayed in key/value format. By default, logs are formatted in a human readable format.", + Hidden: true, Action: func(_ *cli.Context, _ bool) error { opts.LogFormatter.SetFormat(format.NewKeyValueFormat()) @@ -59,6 +63,7 @@ func NewDeprecatedFlags(opts *options.TerragruntOptions) cli.Flags { EnvVar: TerragruntJSONLogEnvName, Destination: &opts.JSONLogFormat, Usage: "If specified, Terragrunt will output its logs in JSON format.", + Hidden: true, Action: func(_ *cli.Context, _ bool) error { opts.LogFormatter.SetFormat(format.NewJSONFormat()) @@ -71,6 +76,24 @@ func NewDeprecatedFlags(opts *options.TerragruntOptions) cli.Flags { opts.Logger.Warnf(warn) } + return nil + }, + }, + &cli.BoolFlag{ + Name: TerragruntTfLogJSONFlagName, + EnvVar: TerragruntTfLogJSONEnvName, + Usage: "If specified, Terragrunt will wrap Terraform stdout and stderr in JSON.", + Hidden: true, + Action: func(_ *cli.Context, _ bool) error { + if control, ok := strict.GetStrictControl(strict.JSONLog); ok { + warn, err := control.Evaluate(opts) + if err != nil { + return err + } + + opts.Logger.Warnf(warn) + } + return nil }, }, diff --git a/config/dependency.go b/config/dependency.go index 9b6eb7f6a..5676cc37f 100644 --- a/config/dependency.go +++ b/config/dependency.go @@ -1051,7 +1051,6 @@ func runTerragruntOutputJSON(ctx *ParsingContext, targetConfig string) ([]byte, newOpts := *ctx.TerragruntOptions // explicit disable json formatting and prefixing to read json output newOpts.ForwardTFStdout = false - newOpts.TerraformLogsToJSON = false newOpts.Writer = stdoutBufferWriter ctx = ctx.WithTerragruntOptions(&newOpts) diff --git a/configstack/running_module.go b/configstack/running_module.go index bd7077499..594ae230c 100644 --- a/configstack/running_module.go +++ b/configstack/running_module.go @@ -140,7 +140,6 @@ func (module *RunningModule) runNow(ctx context.Context, rootOptions *options.Te stdout := bytes.Buffer{} jsonOptions.ForwardTFStdout = true - jsonOptions.TerraformLogsToJSON = false jsonOptions.Writer = &stdout jsonOptions.TerraformCommand = terraform.CommandNameShow jsonOptions.TerraformCliArgs = []string{terraform.CommandNameShow, "-json", module.Module.planFile(rootOptions)} diff --git a/internal/strict/strict.go b/internal/strict/strict.go index 893dd71f0..a59328310 100644 --- a/internal/strict/strict.go +++ b/internal/strict/strict.go @@ -47,6 +47,8 @@ const ( DisableLogFormatting = "terragrunt-disable-log-formatting" // JSONLog is the control that prevents the deprecated `--terragrunt-json-log` flag from being used. JSONLog = "terragrunt-json-log" + // TfLogJSON is the control that prevents the deprecated `--terragrunt-tf-logs-to-json` flag from being used. + TfLogJSON = "terragrunt-tf-logs-to-json" ) // GetStrictControl returns the strict control with the given name. @@ -122,6 +124,10 @@ var StrictControls = Controls{ Error: errors.Errorf("The `--%s` flag is no longer supported. Use `--terragrunt-log-format=json` instead.", JSONLog), Warning: fmt.Sprintf("The `--%s` flag is deprecated and will be removed in a future version. Use `--terragrunt-log-format=json` instead.", JSONLog), }, + TfLogJSON: { + Error: errors.Errorf("The `--%s` flag is no longer supported. Use `--terragrunt-log-format=json` instead.", TfLogJSON), + Warning: fmt.Sprintf("The `--%s` flag is deprecated and will be removed in a future version. Use `--terragrunt-log-format=json` instead.", TfLogJSON), + }, } // Names returns the names of all strict controls. diff --git a/options/options.go b/options/options.go index be9d054b2..1260d7e50 100644 --- a/options/options.go +++ b/options/options.go @@ -147,9 +147,6 @@ type TerragruntOptions struct { // If true, logs will be displayed in formatter key/value, by default logs are formatted in human-readable formatter. DisableLogFormatting bool - // Wrap Terraform logs in JSON format - TerraformLogsToJSON bool - // ValidateStrict mode for the validate-inputs command ValidateStrict bool @@ -469,7 +466,6 @@ func NewTerragruntOptionsWithWriters(stdout, stderr io.Writer) *TerragruntOption ForwardTFStdout: false, JSONOut: DefaultJSONOutName, TerraformImplementation: UnknownImpl, - TerraformLogsToJSON: false, JSONDisableDependentModules: false, RunTerragrunt: func(ctx context.Context, opts *TerragruntOptions) error { return errors.New(ErrRunTerragruntCommandNotSet) @@ -618,7 +614,6 @@ func (opts *TerragruntOptions) Clone(terragruntConfigPath string) (*TerragruntOp FailIfBucketCreationRequired: opts.FailIfBucketCreationRequired, DisableBucketUpdate: opts.DisableBucketUpdate, TerraformImplementation: opts.TerraformImplementation, - TerraformLogsToJSON: opts.TerraformLogsToJSON, GraphRoot: opts.GraphRoot, ScaffoldVars: opts.ScaffoldVars, ScaffoldVarFiles: opts.ScaffoldVarFiles, diff --git a/pkg/log/format/format.go b/pkg/log/format/format.go index 8275a1a6f..747bdd4cf 100644 --- a/pkg/log/format/format.go +++ b/pkg/log/format/format.go @@ -79,9 +79,8 @@ func NewJSONFormat() Placeholders { Level( Escape(JSONEscape), ), - PlainText(`", "prefix":"`), + PlainText(`", "workingDir":"`), Field(WorkDirKeyName, - PathFormat(ShortPath), Escape(JSONEscape), ), PlainText(`", "tfpath":"`), @@ -89,6 +88,10 @@ func NewJSONFormat() Placeholders { PathFormat(FilenamePath), Escape(JSONEscape), ), + PlainText(`", "executedCommandArgs":"`), + Field(TFCmdArgsKeyName, + Escape(JSONEscape), + ), PlainText(`", "msg":"`), Message( PathFormat(RelativePath), diff --git a/pkg/log/format/placeholders/field.go b/pkg/log/format/placeholders/field.go index 6989b7a44..db1386e8c 100644 --- a/pkg/log/format/placeholders/field.go +++ b/pkg/log/format/placeholders/field.go @@ -1,13 +1,16 @@ package placeholders import ( + "strings" + "github.com/gruntwork-io/terragrunt/pkg/log/format/options" ) const ( WorkDirKeyName = "prefix" - DownloadDirKeyName = "downloaddir" - TFPathKeyName = "tfpath" + DownloadDirKeyName = "download-dir" + TFPathKeyName = "tf-path" + TFCmdArgsKeyName = "tf-command-args" ) type fieldPlaceholder struct { @@ -17,8 +20,11 @@ type fieldPlaceholder struct { // Format implements `Placeholder` interface. func (field *fieldPlaceholder) Format(data *options.Data) (string, error) { if val, ok := data.Fields[field.Name()]; ok { - if val, ok := val.(string); ok { + switch val := val.(type) { + case string: return field.opts.Format(data, val) + case []string: + return field.opts.Format(data, strings.Join(val, " ")) } } diff --git a/shell/run_shell_cmd.go b/shell/run_shell_cmd.go index c6c48b98b..0fed82e24 100644 --- a/shell/run_shell_cmd.go +++ b/shell/run_shell_cmd.go @@ -133,25 +133,31 @@ func RunShellCommandWithOutput( errWriter = opts.ErrWriter ) - if opts.JSONLogFormat && opts.TerraformLogsToJSON { - logger := opts.Logger.WithField("workingDir", opts.WorkingDir).WithField("executedCommandArgs", args) - outWriter = logger.WithOptions(log.WithOutput(errWriter)).Writer() - errWriter = logger.WithOptions(log.WithOutput(errWriter)).WriterLevel(log.ErrorLevel) - } else if command == opts.TerraformPath && !opts.TerraformLogsToJSON && !opts.ForwardTFStdout && !shouldForceForwardTFStdout(args) { - logger := opts.Logger.WithField(placeholders.TFPathKeyName, filepath.Base(opts.TerraformPath)) - - outWriter = writer.New( - writer.WithLogger(logger.WithOptions(log.WithOutput(errWriter))), - writer.WithDefaultLevel(log.StdoutLevel), - writer.WithMsgSeparator(logMsgSeparator), - ) - - errWriter = writer.New( - writer.WithLogger(logger.WithOptions(log.WithOutput(errWriter))), - writer.WithDefaultLevel(log.StderrLevel), - writer.WithMsgSeparator(logMsgSeparator), - writer.WithParseFunc(terraform.ParseLogFunc(tfLogMsgPrefix, false)), - ) + if command == opts.TerraformPath { + logger := opts.Logger. + WithField(placeholders.TFPathKeyName, filepath.Base(opts.TerraformPath)). + WithField(placeholders.TFCmdArgsKeyName, args) + + if !opts.ForwardTFStdout && !shouldForceForwardTFStdout(args) { + var msgSeparator string + + if !opts.JSONLogFormat { + msgSeparator = logMsgSeparator + } + + outWriter = writer.New( + writer.WithLogger(logger.WithOptions(log.WithOutput(errWriter))), + writer.WithDefaultLevel(log.StdoutLevel), + writer.WithMsgSeparator(msgSeparator), + ) + + errWriter = writer.New( + writer.WithLogger(logger.WithOptions(log.WithOutput(errWriter))), + writer.WithDefaultLevel(log.StderrLevel), + writer.WithMsgSeparator(msgSeparator), + writer.WithParseFunc(terraform.ParseLogFunc(tfLogMsgPrefix, false)), + ) + } } var ( From 472d78a904e64b4bf3e5fbbf8216ab6eb23b8aad Mon Sep 17 00:00:00 2001 From: Levko Burburas Date: Tue, 26 Nov 2024 21:04:16 +0200 Subject: [PATCH 02/21] chore: fix tests --- config/dependency.go | 1 + configstack/running_module.go | 1 + pkg/log/format/format.go | 21 ++++++++++++++------- pkg/log/format/options/align.go | 4 +++- pkg/log/format/options/case.go | 4 +++- pkg/log/format/options/color.go | 7 +++++-- pkg/log/format/options/content.go | 4 ++-- pkg/log/format/options/escape.go | 6 +++--- pkg/log/format/options/level_format.go | 2 +- pkg/log/format/options/option.go | 10 +++++----- pkg/log/format/options/path_format.go | 6 ++++-- pkg/log/format/options/prefix.go | 4 ++-- pkg/log/format/options/suffix.go | 4 ++-- pkg/log/format/options/time_format.go | 2 +- pkg/log/format/options/util.go | 17 +++++++++++++++++ pkg/log/format/options/width.go | 4 +++- pkg/log/format/placeholders/common.go | 2 +- pkg/log/format/placeholders/field.go | 9 +-------- shell/run_shell_cmd.go | 21 +++++++++++++-------- test/integration_test.go | 2 +- 20 files changed, 83 insertions(+), 48 deletions(-) create mode 100644 pkg/log/format/options/util.go diff --git a/config/dependency.go b/config/dependency.go index 5676cc37f..9e1245e3d 100644 --- a/config/dependency.go +++ b/config/dependency.go @@ -1051,6 +1051,7 @@ func runTerragruntOutputJSON(ctx *ParsingContext, targetConfig string) ([]byte, newOpts := *ctx.TerragruntOptions // explicit disable json formatting and prefixing to read json output newOpts.ForwardTFStdout = false + newOpts.JSONLogFormat = false newOpts.Writer = stdoutBufferWriter ctx = ctx.WithTerragruntOptions(&newOpts) diff --git a/configstack/running_module.go b/configstack/running_module.go index 594ae230c..af41621e3 100644 --- a/configstack/running_module.go +++ b/configstack/running_module.go @@ -140,6 +140,7 @@ func (module *RunningModule) runNow(ctx context.Context, rootOptions *options.Te stdout := bytes.Buffer{} jsonOptions.ForwardTFStdout = true + jsonOptions.JSONLogFormat = false jsonOptions.Writer = &stdout jsonOptions.TerraformCommand = terraform.CommandNameShow jsonOptions.TerraformCliArgs = []string{terraform.CommandNameShow, "-json", module.Module.planFile(rootOptions)} diff --git a/pkg/log/format/format.go b/pkg/log/format/format.go index 747bdd4cf..8fdaa0f3f 100644 --- a/pkg/log/format/format.go +++ b/pkg/log/format/format.go @@ -70,35 +70,42 @@ func NewPrettyFormat() Placeholders { func NewJSONFormat() Placeholders { return Placeholders{ - PlainText(`{"time":"`), + PlainText(`{`), Time( + Prefix(`"time":"`), + Suffix(`"`), TimeFormat(RFC3339), Escape(JSONEscape), ), - PlainText(`", "level":"`), Level( + Prefix(`, "level":"`), + Suffix(`"`), Escape(JSONEscape), ), - PlainText(`", "workingDir":"`), Field(WorkDirKeyName, + Prefix(`, "workingDir":"`), + Suffix(`"`), Escape(JSONEscape), ), - PlainText(`", "tfpath":"`), Field(TFPathKeyName, + Prefix(`, "tfpath":"`), + Suffix(`"`), PathFormat(FilenamePath), Escape(JSONEscape), ), - PlainText(`", "executedCommandArgs":"`), Field(TFCmdArgsKeyName, + Prefix(`, "executedCommandArgs":[`), + Suffix(`]`), Escape(JSONEscape), ), - PlainText(`", "msg":"`), Message( + Prefix(`, "msg":"`), + Suffix(`"`), PathFormat(RelativePath), Color(DisableColor), Escape(JSONEscape), ), - PlainText(`"}`), + PlainText(`}`), } } diff --git a/pkg/log/format/options/align.go b/pkg/log/format/options/align.go index b55522961..a2bc5c214 100644 --- a/pkg/log/format/options/align.go +++ b/pkg/log/format/options/align.go @@ -27,7 +27,9 @@ type AlignOption struct { } // Format implements `Option` interface. -func (option *AlignOption) Format(_ *Data, str string) (string, error) { +func (option *AlignOption) Format(_ *Data, val any) (any, error) { + str := toString(val) + withoutSpaces := strings.TrimSpace(str) spaces := len(str) - len(withoutSpaces) diff --git a/pkg/log/format/options/case.go b/pkg/log/format/options/case.go index c2ee92c0f..f812e8429 100644 --- a/pkg/log/format/options/case.go +++ b/pkg/log/format/options/case.go @@ -30,7 +30,9 @@ type CaseOption struct { } // Format implements `Option` interface. -func (option *CaseOption) Format(_ *Data, str string) (string, error) { +func (option *CaseOption) Format(_ *Data, val any) (any, error) { + str := toString(val) + switch option.value.Get() { case UpperCase: return strings.ToUpper(str), nil diff --git a/pkg/log/format/options/color.go b/pkg/log/format/options/color.go index 12ffe2bd8..1e88fce21 100644 --- a/pkg/log/format/options/color.go +++ b/pkg/log/format/options/color.go @@ -149,8 +149,11 @@ type ColorOption struct { } // Format implements `Option` interface. -func (color *ColorOption) Format(data *Data, str string) (string, error) { - value := color.value.Get() +func (color *ColorOption) Format(data *Data, val any) (any, error) { + var ( + str = toString(val) + value = color.value.Get() + ) if value == NoneColor { return str, nil diff --git a/pkg/log/format/options/content.go b/pkg/log/format/options/content.go index ffc89a199..e91dfdbe8 100644 --- a/pkg/log/format/options/content.go +++ b/pkg/log/format/options/content.go @@ -8,12 +8,12 @@ type ContentOption struct { } // Format implements `Option` interface. -func (option *ContentOption) Format(_ *Data, str string) (string, error) { +func (option *ContentOption) Format(_ *Data, val any) (any, error) { if val := option.value.Get(); val != "" { return val, nil } - return str, nil + return val, nil } // Content creates the option that sets the content. diff --git a/pkg/log/format/options/escape.go b/pkg/log/format/options/escape.go index b27d09029..745116c23 100644 --- a/pkg/log/format/options/escape.go +++ b/pkg/log/format/options/escape.go @@ -25,12 +25,12 @@ type EscapeOption struct { } // Format implements `Option` interface. -func (option *EscapeOption) Format(_ *Data, str string) (string, error) { +func (option *EscapeOption) Format(_ *Data, val any) (any, error) { if option.value.Get() != JSONEscape { - return str, nil + return val, nil } - b, err := json.Marshal(str) + b, err := json.Marshal(val) if err != nil { return "", errors.New(err) } diff --git a/pkg/log/format/options/level_format.go b/pkg/log/format/options/level_format.go index 28821c3fc..f373868e7 100644 --- a/pkg/log/format/options/level_format.go +++ b/pkg/log/format/options/level_format.go @@ -22,7 +22,7 @@ type LevelFormatOption struct { } // Format implements `Option` interface. -func (format *LevelFormatOption) Format(data *Data, _ string) (string, error) { +func (format *LevelFormatOption) Format(data *Data, _ any) (any, error) { switch format.value.Get() { case LevelFormatTiny: return data.Level.TinyName(), nil diff --git a/pkg/log/format/options/option.go b/pkg/log/format/options/option.go index 75d92e6a3..8ded8a60e 100644 --- a/pkg/log/format/options/option.go +++ b/pkg/log/format/options/option.go @@ -20,7 +20,7 @@ type Option interface { // Name returns the name of the option. Name() string // Format formats the given string. - Format(data *Data, str string) (string, error) + Format(data *Data, val any) (any, error) // ParseValue parses and sets the value of the option. ParseValue(str string) error } @@ -76,15 +76,15 @@ func (opts Options) Merge(withOpts ...Option) Options { } // Format returns the formatted value. -func (opts Options) Format(data *Data, str string) (string, error) { +func (opts Options) Format(data *Data, val any) (string, error) { var err error for _, opt := range opts { - str, err = opt.Format(data, str) - if str == "" || err != nil { + val, err = opt.Format(data, val) + if val == "" || err != nil { return "", err } } - return str, nil + return toString(val), nil } diff --git a/pkg/log/format/options/path_format.go b/pkg/log/format/options/path_format.go index 6f0465ca6..1aa4f426c 100644 --- a/pkg/log/format/options/path_format.go +++ b/pkg/log/format/options/path_format.go @@ -38,7 +38,9 @@ type PathFormatOption struct { } // Format implements `Option` interface. -func (option *PathFormatOption) Format(data *Data, str string) (string, error) { +func (option *PathFormatOption) Format(data *Data, val any) (any, error) { + str := toString(val) + switch option.value.Get() { case RelativePath: if data.RelativePather == nil { @@ -75,7 +77,7 @@ func (option *PathFormatOption) Format(data *Data, str string) (string, error) { case NonePath: } - return str, nil + return val, nil } // PathFormat creates the option to format the paths. diff --git a/pkg/log/format/options/prefix.go b/pkg/log/format/options/prefix.go index 9f19a1d52..5b82fa0ff 100644 --- a/pkg/log/format/options/prefix.go +++ b/pkg/log/format/options/prefix.go @@ -8,8 +8,8 @@ type PrefixOption struct { } // Format implements `Option` interface. -func (option *PrefixOption) Format(_ *Data, str string) (string, error) { - return option.value.Get() + str, nil +func (option *PrefixOption) Format(_ *Data, val any) (any, error) { + return option.value.Get() + toString(val), nil } // Prefix creates the option to add a prefix to the text. diff --git a/pkg/log/format/options/suffix.go b/pkg/log/format/options/suffix.go index 25194bedf..c0d54b640 100644 --- a/pkg/log/format/options/suffix.go +++ b/pkg/log/format/options/suffix.go @@ -8,8 +8,8 @@ type SuffixOption struct { } // Format implements `Option` interface. -func (option *SuffixOption) Format(_ *Data, str string) (string, error) { - return str + option.value.Get(), nil +func (option *SuffixOption) Format(_ *Data, val any) (any, error) { + return toString(val) + option.value.Get(), nil } // Suffix creates the option to add a suffix to the text. diff --git a/pkg/log/format/options/time_format.go b/pkg/log/format/options/time_format.go index d77afa2f0..5ad85e890 100644 --- a/pkg/log/format/options/time_format.go +++ b/pkg/log/format/options/time_format.go @@ -115,7 +115,7 @@ type TimeFormatOption struct { } // Format implements `Option` interface. -func (option *TimeFormatOption) Format(data *Data, _ string) (string, error) { +func (option *TimeFormatOption) Format(data *Data, _ any) (any, error) { return data.Time.Format(option.value.Get()), nil } diff --git a/pkg/log/format/options/util.go b/pkg/log/format/options/util.go new file mode 100644 index 000000000..7692fa759 --- /dev/null +++ b/pkg/log/format/options/util.go @@ -0,0 +1,17 @@ +package options + +import ( + "fmt" + "strings" +) + +func toString(val any) string { + switch val := val.(type) { + case string: + return val + case []string: + return strings.Join(val, " ") + } + + return fmt.Sprintf("%v", val) +} diff --git a/pkg/log/format/options/width.go b/pkg/log/format/options/width.go index 199ff0ac9..07d38bf61 100644 --- a/pkg/log/format/options/width.go +++ b/pkg/log/format/options/width.go @@ -14,7 +14,9 @@ type WidthOption struct { } // Format implements `Option` interface. -func (option *WidthOption) Format(_ *Data, str string) (string, error) { +func (option *WidthOption) Format(_ *Data, val any) (any, error) { + str := toString(val) + width := option.value.Get() if width == 0 { return str, nil diff --git a/pkg/log/format/placeholders/common.go b/pkg/log/format/placeholders/common.go index b37766928..8b5a5afb3 100644 --- a/pkg/log/format/placeholders/common.go +++ b/pkg/log/format/placeholders/common.go @@ -11,12 +11,12 @@ import ( func WithCommonOptions(opts ...options.Option) options.Options { return options.Options(append(opts, options.Content(""), + options.Escape(options.NoneEscape), options.Case(options.NoneCase), options.Width(0), options.Align(options.NoneAlign), options.Prefix(""), options.Suffix(""), - options.Escape(options.NoneEscape), options.Color(options.NoneColor), )) } diff --git a/pkg/log/format/placeholders/field.go b/pkg/log/format/placeholders/field.go index db1386e8c..8008b15b4 100644 --- a/pkg/log/format/placeholders/field.go +++ b/pkg/log/format/placeholders/field.go @@ -1,8 +1,6 @@ package placeholders import ( - "strings" - "github.com/gruntwork-io/terragrunt/pkg/log/format/options" ) @@ -20,12 +18,7 @@ type fieldPlaceholder struct { // Format implements `Placeholder` interface. func (field *fieldPlaceholder) Format(data *options.Data) (string, error) { if val, ok := data.Fields[field.Name()]; ok { - switch val := val.(type) { - case string: - return field.opts.Format(data, val) - case []string: - return field.opts.Format(data, strings.Join(val, " ")) - } + return field.opts.Format(data, val) } return "", nil diff --git a/shell/run_shell_cmd.go b/shell/run_shell_cmd.go index 0fed82e24..118f888f2 100644 --- a/shell/run_shell_cmd.go +++ b/shell/run_shell_cmd.go @@ -133,28 +133,33 @@ func RunShellCommandWithOutput( errWriter = opts.ErrWriter ) - if command == opts.TerraformPath { + if command == opts.TerraformPath && !opts.ForwardTFStdout { logger := opts.Logger. WithField(placeholders.TFPathKeyName, filepath.Base(opts.TerraformPath)). WithField(placeholders.TFCmdArgsKeyName, args) - if !opts.ForwardTFStdout && !shouldForceForwardTFStdout(args) { - var msgSeparator string + if opts.JSONLogFormat { + outWriter = writer.New( + writer.WithLogger(logger.WithOptions(log.WithOutput(errWriter))), + writer.WithDefaultLevel(log.StdoutLevel), + ) - if !opts.JSONLogFormat { - msgSeparator = logMsgSeparator - } + errWriter = writer.New( + writer.WithLogger(logger.WithOptions(log.WithOutput(errWriter))), + writer.WithDefaultLevel(log.StderrLevel), + ) + } else if !shouldForceForwardTFStdout(args) { outWriter = writer.New( writer.WithLogger(logger.WithOptions(log.WithOutput(errWriter))), writer.WithDefaultLevel(log.StdoutLevel), - writer.WithMsgSeparator(msgSeparator), + writer.WithMsgSeparator(logMsgSeparator), ) errWriter = writer.New( writer.WithLogger(logger.WithOptions(log.WithOutput(errWriter))), writer.WithDefaultLevel(log.StderrLevel), - writer.WithMsgSeparator(msgSeparator), + writer.WithMsgSeparator(logMsgSeparator), writer.WithParseFunc(terraform.ParseLogFunc(tfLogMsgPrefix, false)), ) } diff --git a/test/integration_test.go b/test/integration_test.go index 5ae1323bf..bf90cccc2 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -3765,7 +3765,7 @@ func TestTerragruntTerraformOutputJson(t *testing.T) { helpers.CleanupTerraformFolder(t, tmpEnvPath) testPath := util.JoinPath(tmpEnvPath, testFixtureInitError) - _, stderr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt apply --no-color --terragrunt-json-log --terragrunt-tf-logs-to-json --terragrunt-forward-tf-stdout --terragrunt-non-interactive --terragrunt-working-dir "+testPath) + _, stderr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt apply --no-color --terragrunt-json-log --terragrunt-tf-logs-to-json --terragrunt-non-interactive --terragrunt-working-dir "+testPath) require.Error(t, err) // Sometimes, this is the error returned by AWS. From 6930bf83daeac2da2854cfc47c60a4b5974d6783 Mon Sep 17 00:00:00 2001 From: Levko Burburas Date: Tue, 26 Nov 2024 21:56:20 +0200 Subject: [PATCH 03/21] chore: fix tests --- docs/_docs/02_features/custom-log-format.md | 10 +++++----- docs/_docs/04_reference/cli-options.md | 6 ++++-- shell/run_shell_cmd.go | 1 - test/integration_test.go | 5 ++++- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/docs/_docs/02_features/custom-log-format.md b/docs/_docs/02_features/custom-log-format.md index 882cdd667..3546a3790 100644 --- a/docs/_docs/02_features/custom-log-format.md +++ b/docs/_docs/02_features/custom-log-format.md @@ -56,7 +56,7 @@ Placeholders have preset names: * `%prefix` - Path to the working directory were Terragrunt is running. -* `%tfpath` - Path to the OpenTofu/Terraform executable (as defined by [terragrunt-tfpath](https://terragrunt.gruntwork.io/docs/reference/cli-options/#terragrunt-tfpath)). +* `%tf-path` - Path to the OpenTofu/Terraform executable (as defined by [terragrunt-tfpath](https://terragrunt.gruntwork.io/docs/reference/cli-options/#terragrunt-tfpath)). * `%msg` - Log message. @@ -256,7 +256,7 @@ Specific options for placeholders: * `short` - Outputs an absolute path, but hides the working directory path. -* `%tfpath` +* `%tf-path` * `path=[filename|dir]` @@ -277,7 +277,7 @@ The examples below replicate the preset formats specified with `--terragrunt-log `--terragrunt-log-format pretty` ```shell ---terragrunt-log-custom-format "%time(color=light-black) %level(case=upper,width=6,color=preset) %prefix(path=short-relative,color=gradient,suffix=' ')%tfpath(color=cyan,suffix=': ')%msg(path=relative)" +--terragrunt-log-custom-format "%time(color=light-black) %level(case=upper,width=6,color=preset) %prefix(path=short-relative,color=gradient,suffix=' ')%tf-path(color=cyan,suffix=': ')%msg(path=relative)" ``` `--terragrunt-log-format bare` @@ -289,11 +289,11 @@ The examples below replicate the preset formats specified with `--terragrunt-log `--terragrunt-log-format key-value` ```shell ---terragrunt-log-custom-format "time=%time(format=rfc3339) level=%level prefix=%prefix(path=short-relative) tfpath=%tfpath(path=filename) msg=%msg(path=relative,color=disable)" +--terragrunt-log-custom-format "time=%time(format=rfc3339) level=%level prefix=%prefix(path=short-relative) tfpath=%tf-path(path=filename) msg=%msg(path=relative,color=disable)" ``` `--terragrunt-log-format json` ```shell ---terragrunt-log-custom-format '{"time":"%time(format=rfc3339,escape=json)", "level":"%level(escape=json)", "prefix":"%prefix(path=short-relative,escape=json)", "tfpath":"%tfpath(path=filename,escape=json)", "msg":"%msg(path=relative,escape=json,color=disable)"}' +--terragrunt-log-custom-format '{"time":"%time(format=rfc3339,escape=json)", "level":"%level(escape=json)", "prefix":"%prefix(path=short-relative,escape=json)", "tfpath":"%tf-path(path=filename,escape=json)", "msg":"%msg(path=relative,escape=json,color=disable)"}' ``` diff --git a/docs/_docs/04_reference/cli-options.md b/docs/_docs/04_reference/cli-options.md index 35e9343d8..1c132977c 100644 --- a/docs/_docs/04_reference/cli-options.md +++ b/docs/_docs/04_reference/cli-options.md @@ -82,7 +82,7 @@ This page documents the CLI commands and options available with Terragrunt: - [terragrunt-disable-bucket-update](#terragrunt-disable-bucket-update) - [terragrunt-disable-command-validation](#terragrunt-disable-command-validation) - [terragrunt-json-log](#terragrunt-json-log) (DEPRECATED: use [terragrunt-log-format](#terragrunt-log-format)) - - [terragrunt-tf-logs-to-json](#terragrunt-tf-logs-to-json) + - [terragrunt-tf-logs-to-json](#terragrunt-tf-logs-to-json) (DEPRECATED: use [terragrunt-log-format](#terragrunt-log-format)) - [terragrunt-provider-cache](#terragrunt-provider-cache) - [terragrunt-provider-cache-dir](#terragrunt-provider-cache-dir) - [terragrunt-provider-cache-hostname](#terragrunt-provider-cache-hostname) @@ -809,7 +809,7 @@ prefix `--terragrunt-` (e.g., `--terragrunt-config`). The currently available op - [terragrunt-disable-bucket-update](#terragrunt-disable-bucket-update) - [terragrunt-disable-command-validation](#terragrunt-disable-command-validation) - [terragrunt-json-log](#terragrunt-json-log) (DEPRECATED: use [terragrunt-log-format](#terragrunt-log-format)) - - [terragrunt-tf-logs-to-json](#terragrunt-tf-logs-to-json) + - [terragrunt-tf-logs-to-json](#terragrunt-tf-logs-to-json) (DEPRECATED: use [terragrunt-log-format](#terragrunt-log-format)) - [terragrunt-provider-cache](#terragrunt-provider-cache) - [terragrunt-provider-cache-dir](#terragrunt-provider-cache-dir) - [terragrunt-provider-cache-hostname](#terragrunt-provider-cache-hostname) @@ -1485,6 +1485,8 @@ When this flag is set, Terragrunt will output its logs in JSON format. ### terragrunt-tf-logs-to-json +DEPRECATED: Use [terragrunt-log-format](#terragrunt-log-format). OpenTofu/Terraform `stdout` and `stderr` is wrapped in JSON by default with `--terragurnt-log-format json` flag if `--terragrunt-forward-tf-stdout` flag is not specified. + **CLI Arg**: `--terragrunt-tf-logs-to-json`
**Environment Variable**: `TERRAGRUNT_TF_JSON_LOG` (set to `true`)
diff --git a/shell/run_shell_cmd.go b/shell/run_shell_cmd.go index 118f888f2..cee2a9372 100644 --- a/shell/run_shell_cmd.go +++ b/shell/run_shell_cmd.go @@ -148,7 +148,6 @@ func RunShellCommandWithOutput( writer.WithLogger(logger.WithOptions(log.WithOutput(errWriter))), writer.WithDefaultLevel(log.StderrLevel), ) - } else if !shouldForceForwardTFStdout(args) { outWriter = writer.New( writer.WithLogger(logger.WithOptions(log.WithOutput(errWriter))), diff --git a/test/integration_test.go b/test/integration_test.go index bf90cccc2..884fc3908 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -215,7 +215,7 @@ func TestLogCustomFormatOutput(t *testing.T) { }, }, { - logCustomFormat: "%interval%(content=' plain-text ')%level(case=upper,width=6) %prefix(path=short-relative,suffix=' ')%tfpath(suffix=': ')%msg(path=relative)", + logCustomFormat: "%interval%(content=' plain-text ')%level(case=upper,width=6) %prefix(path=short-relative,suffix=' ')%tf-path(suffix=': ')%msg(path=relative)", expectedOutputRegs: []*regexp.Regexp{ regexp.MustCompile(`\d{4}` + regexp.QuoteMeta(" plain-text DEBUG Terragrunt Version:")), regexp.MustCompile(`\d{4}` + regexp.QuoteMeta(" plain-text STDOUT dep "+wrappedBinary()+": Initializing the backend...")), @@ -234,6 +234,9 @@ func TestLogCustomFormatOutput(t *testing.T) { tmpEnvPath := helpers.CopyEnvironment(t, testFixtureLogFormatter) rootPath := util.JoinPath(tmpEnvPath, testFixtureLogFormatter) + rootPath, err := filepath.EvalSymlinks(rootPath) + require.NoError(t, err) + _, stderr, err := helpers.RunTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt run-all init --terragrunt-log-level debug --terragrunt-non-interactive -no-color --terragrunt-no-color --terragrunt-log-custom-format=%q --terragrunt-working-dir %s", testCase.logCustomFormat, rootPath)) require.NoError(t, err) From 8f0ffdc059546f0600ab8d54b6728dcb9d68399a Mon Sep 17 00:00:00 2001 From: Levko Burburas Date: Tue, 26 Nov 2024 22:25:35 +0200 Subject: [PATCH 04/21] chore: fix tests --- cli/provider_cache.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cli/provider_cache.go b/cli/provider_cache.go index 5d74382b3..42ce7ab1f 100644 --- a/cli/provider_cache.go +++ b/cli/provider_cache.go @@ -350,7 +350,6 @@ func runTerraformCommand(ctx context.Context, opts *options.TerragruntOptions, a cloneOpts.WorkingDir = opts.WorkingDir cloneOpts.TerraformCliArgs = args cloneOpts.Env = envs - cloneOpts.ForwardTFStdout = true // If the Terraform error matches `httpStatusCacheProviderReg` we ignore it and hide the log from users, otherwise we process the error as is. if output, err := shell.RunTerraformCommandWithOutput(ctx, cloneOpts, cloneOpts.TerraformCliArgs...); err != nil && len(errWriter.Msgs()) == 0 { From ec12ed33201b1212df57b6e7aeb1ba5f88900c55 Mon Sep 17 00:00:00 2001 From: Levko Burburas Date: Tue, 26 Nov 2024 23:51:18 +0200 Subject: [PATCH 05/21] fix: provider cache --- cli/provider_cache.go | 12 +++++++++--- util/trap_writer.go | 42 ++++++++++++++++-------------------------- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/cli/provider_cache.go b/cli/provider_cache.go index 42ce7ab1f..a1694bc51 100644 --- a/cli/provider_cache.go +++ b/cli/provider_cache.go @@ -333,7 +333,7 @@ func (cache *ProviderCache) createLocalCLIConfig(ctx context.Context, opts *opti func runTerraformCommand(ctx context.Context, opts *options.TerragruntOptions, args []string, envs map[string]string) (*util.CmdOutput, error) { // We use custom writer in order to trap the log from `terraform providers lock -platform=provider-cache` command, which terraform considers an error, but to us a success. - errWriter := util.NewTrapWriter(opts.ErrWriter, httpStatusCacheProviderReg) + errWriter := util.NewTrapWriter(opts.ErrWriter) // add -no-color flag to args if it was set in Terragrunt arguments if util.ListContainsElement(opts.TerraformCliArgs, terraform.FlagNameNoColor) { @@ -352,8 +352,14 @@ func runTerraformCommand(ctx context.Context, opts *options.TerragruntOptions, a cloneOpts.Env = envs // If the Terraform error matches `httpStatusCacheProviderReg` we ignore it and hide the log from users, otherwise we process the error as is. - if output, err := shell.RunTerraformCommandWithOutput(ctx, cloneOpts, cloneOpts.TerraformCliArgs...); err != nil && len(errWriter.Msgs()) == 0 { - return output, err + if output, err := shell.RunTerraformCommandWithOutput(ctx, cloneOpts, cloneOpts.TerraformCliArgs...); err != nil && !httpStatusCacheProviderReg.Match(output.Stderr.Bytes()) { + errs := new(errors.MultiError).Append(err) + + if err := errWriter.Flush(); err != nil { + errs = errs.Append(err) + } + + return output, errs } return nil, nil diff --git a/util/trap_writer.go b/util/trap_writer.go index 4bdb2d3f0..c6c89d469 100644 --- a/util/trap_writer.go +++ b/util/trap_writer.go @@ -2,48 +2,38 @@ package util import ( "io" - "regexp" -) - -const ansi = "[\u001B\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))" -// regexp matches ansi characters getting from a shell output, used for colors etc. -var ansiReg = regexp.MustCompile(ansi) + "github.com/gruntwork-io/terragrunt/internal/errors" +) -// TrapWriter intercepts any messages matching `reg` received from the `writer` output, but passes all others. +// TrapWriter intercepts any messages received from the `writer` output. // Used when necessary to filter logs from terraform. type TrapWriter struct { - writer io.Writer - reg *regexp.Regexp - trappedMsgs []string + writer io.Writer + msgs [][]byte } // NewTrapWriter returns a new TrapWriter instance. -func NewTrapWriter(writer io.Writer, reg *regexp.Regexp) *TrapWriter { +func NewTrapWriter(writer io.Writer) *TrapWriter { return &TrapWriter{ writer: writer, - reg: reg, } } -// Msgs returns the intercepted messages. -func (trap *TrapWriter) Msgs() []string { - return trap.trappedMsgs -} +// Flush flushes intercepted messages to the writer. +func (trap *TrapWriter) Flush() error { + for _, msg := range trap.msgs { + if _, err := trap.writer.Write(msg); err != nil { + return errors.New(err) + } + } -// Clear clears all intercepted messages. -func (trap *TrapWriter) Clear() { - trap.trappedMsgs = nil + return nil } // Write implements `io.Writer` interface. func (trap *TrapWriter) Write(msg []byte) (int, error) { - msgWithoutAnsi := ansiReg.ReplaceAll(msg, []byte("")) - - if trap.reg.Match(msgWithoutAnsi) { - trap.trappedMsgs = append(trap.trappedMsgs, string(msg)) - return len(msg), nil - } + trap.msgs = append(trap.msgs, msg) - return trap.writer.Write(msg) + return len(msg), nil } From 72fcfd1c105751f38252a7d31f249be85f33b857 Mon Sep 17 00:00:00 2001 From: Levko Burburas Date: Wed, 27 Nov 2024 00:02:15 +0200 Subject: [PATCH 06/21] chore: fix tests --- test/integration_strict_test.go | 29 +++++++++++++++++++++++++++++ test/integration_test.go | 29 ----------------------------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/test/integration_strict_test.go b/test/integration_strict_test.go index dd656b808..76cbfaad5 100644 --- a/test/integration_strict_test.go +++ b/test/integration_strict_test.go @@ -1,6 +1,8 @@ package test_test import ( + "encoding/json" + "strings" "testing" "github.com/gruntwork-io/terragrunt/internal/strict" @@ -81,3 +83,30 @@ func TestStrictMode(t *testing.T) { }) } } + +func TestTerragruntTerraformOutputJson(t *testing.T) { + tmpEnvPath := helpers.CopyEnvironment(t, testFixtureInitError) + helpers.CleanupTerraformFolder(t, tmpEnvPath) + testPath := util.JoinPath(tmpEnvPath, testFixtureInitError) + + _, stderr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt apply --no-color --terragrunt-json-log --terragrunt-tf-logs-to-json --terragrunt-non-interactive --terragrunt-working-dir "+testPath) + require.Error(t, err) + + // Sometimes, this is the error returned by AWS. + if !strings.Contains(stderr, "Error: Failed to get existing workspaces: operation error S3: ListObjectsV2, https response error StatusCode: 301") { + assert.Contains(t, stderr, `"msg":"Initializing the backend..."`) + } + + // check if output can be extracted in json + jsonStrings := strings.Split(stderr, "\n") + for _, jsonString := range jsonStrings { + if len(jsonString) == 0 { + continue + } + var output map[string]interface{} + err = json.Unmarshal([]byte(jsonString), &output) + require.NoErrorf(t, err, "Failed to parse json %s", jsonString) + assert.NotNil(t, output["level"]) + assert.NotNil(t, output["time"]) + } +} diff --git a/test/integration_test.go b/test/integration_test.go index 884fc3908..5cd20a110 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -3761,35 +3761,6 @@ func TestLogFormatJSONOutput(t *testing.T) { } } -func TestTerragruntTerraformOutputJson(t *testing.T) { - t.Parallel() - - tmpEnvPath := helpers.CopyEnvironment(t, testFixtureInitError) - helpers.CleanupTerraformFolder(t, tmpEnvPath) - testPath := util.JoinPath(tmpEnvPath, testFixtureInitError) - - _, stderr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt apply --no-color --terragrunt-json-log --terragrunt-tf-logs-to-json --terragrunt-non-interactive --terragrunt-working-dir "+testPath) - require.Error(t, err) - - // Sometimes, this is the error returned by AWS. - if !strings.Contains(stderr, "Error: Failed to get existing workspaces: operation error S3: ListObjectsV2, https response error StatusCode: 301") { - assert.Contains(t, stderr, `"msg":"Initializing the backend..."`) - } - - // check if output can be extracted in json - jsonStrings := strings.Split(stderr, "\n") - for _, jsonString := range jsonStrings { - if len(jsonString) == 0 { - continue - } - var output map[string]interface{} - err = json.Unmarshal([]byte(jsonString), &output) - require.NoErrorf(t, err, "Failed to parse json %s", jsonString) - assert.NotNil(t, output["level"]) - assert.NotNil(t, output["time"]) - } -} - func TestTerragruntOutputFromDependencyLogsJson(t *testing.T) { t.Parallel() From 21b9663f8a1b83f4ecde47928d22b265244f30bb Mon Sep 17 00:00:00 2001 From: Levko Burburas Date: Wed, 27 Nov 2024 00:10:48 +0200 Subject: [PATCH 07/21] chore: fix test --- test/integration_strict_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/integration_strict_test.go b/test/integration_strict_test.go index 76cbfaad5..a67f711ec 100644 --- a/test/integration_strict_test.go +++ b/test/integration_strict_test.go @@ -2,6 +2,7 @@ package test_test import ( "encoding/json" + "regexp" "strings" "testing" @@ -94,7 +95,7 @@ func TestTerragruntTerraformOutputJson(t *testing.T) { // Sometimes, this is the error returned by AWS. if !strings.Contains(stderr, "Error: Failed to get existing workspaces: operation error S3: ListObjectsV2, https response error StatusCode: 301") { - assert.Contains(t, stderr, `"msg":"Initializing the backend..."`) + assert.Regexp(t, stderr, `"msg":".*`+regexp.QuoteMeta("Initializing the backend...")) } // check if output can be extracted in json From abfcfd6785430a4bc7687f195deb38f5f29cdaa8 Mon Sep 17 00:00:00 2001 From: Levko Burburas Date: Wed, 27 Nov 2024 00:19:48 +0200 Subject: [PATCH 08/21] chore: fix test --- cli/provider_cache.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/cli/provider_cache.go b/cli/provider_cache.go index a1694bc51..9504bedd3 100644 --- a/cli/provider_cache.go +++ b/cli/provider_cache.go @@ -353,13 +353,11 @@ func runTerraformCommand(ctx context.Context, opts *options.TerragruntOptions, a // If the Terraform error matches `httpStatusCacheProviderReg` we ignore it and hide the log from users, otherwise we process the error as is. if output, err := shell.RunTerraformCommandWithOutput(ctx, cloneOpts, cloneOpts.TerraformCliArgs...); err != nil && !httpStatusCacheProviderReg.Match(output.Stderr.Bytes()) { - errs := new(errors.MultiError).Append(err) - - if err := errWriter.Flush(); err != nil { - errs = errs.Append(err) - } + return output, err + } - return output, errs + if err := errWriter.Flush(); err != nil { + return nil, err } return nil, nil From c15ddfed2374633eb286cfbfa81e13e477926b29 Mon Sep 17 00:00:00 2001 From: Levko Burburas Date: Wed, 27 Nov 2024 00:21:57 +0200 Subject: [PATCH 09/21] chore: fix test --- test/integration_serial_test.go | 28 ++++++++++++++++++++++++++++ test/integration_strict_test.go | 30 ------------------------------ 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/test/integration_serial_test.go b/test/integration_serial_test.go index d12442340..a2cb0f3ba 100644 --- a/test/integration_serial_test.go +++ b/test/integration_serial_test.go @@ -11,6 +11,7 @@ import ( "os" "path" "path/filepath" + "regexp" "runtime" "strings" "testing" @@ -705,3 +706,30 @@ func TestParseTFLog(t *testing.T) { assert.Contains(t, stderr, "INFO ["+prefixName+"] "+wrappedBinary()+`: TF_LOG: Go runtime version`) } } + +func TestTerragruntTerraformOutputJson(t *testing.T) { + tmpEnvPath := helpers.CopyEnvironment(t, testFixtureInitError) + helpers.CleanupTerraformFolder(t, tmpEnvPath) + testPath := util.JoinPath(tmpEnvPath, testFixtureInitError) + + _, stderr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt apply --no-color --terragrunt-json-log --terragrunt-tf-logs-to-json --terragrunt-non-interactive --terragrunt-working-dir "+testPath) + require.Error(t, err) + + // Sometimes, this is the error returned by AWS. + if !strings.Contains(stderr, "Error: Failed to get existing workspaces: operation error S3: ListObjectsV2, https response error StatusCode: 301") { + assert.Regexp(t, `"msg":".*`+regexp.QuoteMeta("Initializing the backend..."), stderr) + } + + // check if output can be extracted in json + jsonStrings := strings.Split(stderr, "\n") + for _, jsonString := range jsonStrings { + if len(jsonString) == 0 { + continue + } + var output map[string]interface{} + err = json.Unmarshal([]byte(jsonString), &output) + require.NoErrorf(t, err, "Failed to parse json %s", jsonString) + assert.NotNil(t, output["level"]) + assert.NotNil(t, output["time"]) + } +} diff --git a/test/integration_strict_test.go b/test/integration_strict_test.go index a67f711ec..dd656b808 100644 --- a/test/integration_strict_test.go +++ b/test/integration_strict_test.go @@ -1,9 +1,6 @@ package test_test import ( - "encoding/json" - "regexp" - "strings" "testing" "github.com/gruntwork-io/terragrunt/internal/strict" @@ -84,30 +81,3 @@ func TestStrictMode(t *testing.T) { }) } } - -func TestTerragruntTerraformOutputJson(t *testing.T) { - tmpEnvPath := helpers.CopyEnvironment(t, testFixtureInitError) - helpers.CleanupTerraformFolder(t, tmpEnvPath) - testPath := util.JoinPath(tmpEnvPath, testFixtureInitError) - - _, stderr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt apply --no-color --terragrunt-json-log --terragrunt-tf-logs-to-json --terragrunt-non-interactive --terragrunt-working-dir "+testPath) - require.Error(t, err) - - // Sometimes, this is the error returned by AWS. - if !strings.Contains(stderr, "Error: Failed to get existing workspaces: operation error S3: ListObjectsV2, https response error StatusCode: 301") { - assert.Regexp(t, stderr, `"msg":".*`+regexp.QuoteMeta("Initializing the backend...")) - } - - // check if output can be extracted in json - jsonStrings := strings.Split(stderr, "\n") - for _, jsonString := range jsonStrings { - if len(jsonString) == 0 { - continue - } - var output map[string]interface{} - err = json.Unmarshal([]byte(jsonString), &output) - require.NoErrorf(t, err, "Failed to parse json %s", jsonString) - assert.NotNil(t, output["level"]) - assert.NotNil(t, output["time"]) - } -} From ff18d1ed2ae9c9bd2bf0b2f0b7b5ecdd5338a22e Mon Sep 17 00:00:00 2001 From: Levko Burburas Date: Wed, 27 Nov 2024 00:29:23 +0200 Subject: [PATCH 10/21] chore: fix test --- cli/provider_cache.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cli/provider_cache.go b/cli/provider_cache.go index 9504bedd3..89450f420 100644 --- a/cli/provider_cache.go +++ b/cli/provider_cache.go @@ -351,16 +351,17 @@ func runTerraformCommand(ctx context.Context, opts *options.TerragruntOptions, a cloneOpts.TerraformCliArgs = args cloneOpts.Env = envs + output, err := shell.RunTerraformCommandWithOutput(ctx, cloneOpts, cloneOpts.TerraformCliArgs...) // If the Terraform error matches `httpStatusCacheProviderReg` we ignore it and hide the log from users, otherwise we process the error as is. - if output, err := shell.RunTerraformCommandWithOutput(ctx, cloneOpts, cloneOpts.TerraformCliArgs...); err != nil && !httpStatusCacheProviderReg.Match(output.Stderr.Bytes()) { - return output, err + if httpStatusCacheProviderReg.Match(output.Stderr.Bytes()) { + return nil, nil } if err := errWriter.Flush(); err != nil { return nil, err } - return nil, nil + return output, err } // providerCacheEnvironment returns TF_* name/value ENVs, which we use to force terraform processes to make requests through our cache server (proxy) instead of making direct requests to the origin servers. From 812576768cf2f378628cbd71006288d6dba1d1c0 Mon Sep 17 00:00:00 2001 From: Levko Burburas Date: Wed, 27 Nov 2024 00:33:59 +0200 Subject: [PATCH 11/21] chore: fix test --- cli/provider_cache.go | 2 +- pkg/log/writer/writer.go | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cli/provider_cache.go b/cli/provider_cache.go index 89450f420..33648d0a7 100644 --- a/cli/provider_cache.go +++ b/cli/provider_cache.go @@ -353,7 +353,7 @@ func runTerraformCommand(ctx context.Context, opts *options.TerragruntOptions, a output, err := shell.RunTerraformCommandWithOutput(ctx, cloneOpts, cloneOpts.TerraformCliArgs...) // If the Terraform error matches `httpStatusCacheProviderReg` we ignore it and hide the log from users, otherwise we process the error as is. - if httpStatusCacheProviderReg.Match(output.Stderr.Bytes()) { + if err != nil && httpStatusCacheProviderReg.Match(output.Stderr.Bytes()) { return nil, nil } diff --git a/pkg/log/writer/writer.go b/pkg/log/writer/writer.go index f349681f7..8dd241a8e 100644 --- a/pkg/log/writer/writer.go +++ b/pkg/log/writer/writer.go @@ -3,6 +3,7 @@ package writer import ( "strings" + "sync" "time" "github.com/gruntwork-io/terragrunt/pkg/log" @@ -17,6 +18,7 @@ type Writer struct { defaultLevel log.Level msgSeparator string parseFunc WriterParseFunc + mu sync.Mutex } // New returns a new Writer instance with fields assigned to default values. @@ -40,6 +42,9 @@ func (writer *Writer) SetOption(opts ...Option) { // Write implements `io.Writer` interface. func (writer *Writer) Write(p []byte) (n int, err error) { + writer.mu.Lock() + defer writer.mu.Unlock() + var ( str = string(p) strs = []string{str} From e10514f9007d5766185d5c5af9486b009bbd8ee0 Mon Sep 17 00:00:00 2001 From: Levko Burburas Date: Wed, 27 Nov 2024 00:44:13 +0200 Subject: [PATCH 12/21] chore: fix test --- cli/provider_cache.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cli/provider_cache.go b/cli/provider_cache.go index 33648d0a7..9e7e3b834 100644 --- a/cli/provider_cache.go +++ b/cli/provider_cache.go @@ -353,12 +353,14 @@ func runTerraformCommand(ctx context.Context, opts *options.TerragruntOptions, a output, err := shell.RunTerraformCommandWithOutput(ctx, cloneOpts, cloneOpts.TerraformCliArgs...) // If the Terraform error matches `httpStatusCacheProviderReg` we ignore it and hide the log from users, otherwise we process the error as is. - if err != nil && httpStatusCacheProviderReg.Match(output.Stderr.Bytes()) { - return nil, nil - } + if err != nil { + if httpStatusCacheProviderReg.Match(output.Stderr.Bytes()) { + return nil, nil + } - if err := errWriter.Flush(); err != nil { - return nil, err + if err := errWriter.Flush(); err != nil { + return nil, err + } } return output, err From 962cc8dffb2080abb66d66eeef095c8431113bc5 Mon Sep 17 00:00:00 2001 From: Levko Burburas Date: Wed, 27 Nov 2024 01:02:19 +0200 Subject: [PATCH 13/21] chore: fix test --- cli/provider_cache.go | 12 +++++------- pkg/log/format/formatter.go | 5 +++++ pkg/log/writer/writer.go | 5 ----- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/cli/provider_cache.go b/cli/provider_cache.go index 9e7e3b834..33648d0a7 100644 --- a/cli/provider_cache.go +++ b/cli/provider_cache.go @@ -353,14 +353,12 @@ func runTerraformCommand(ctx context.Context, opts *options.TerragruntOptions, a output, err := shell.RunTerraformCommandWithOutput(ctx, cloneOpts, cloneOpts.TerraformCliArgs...) // If the Terraform error matches `httpStatusCacheProviderReg` we ignore it and hide the log from users, otherwise we process the error as is. - if err != nil { - if httpStatusCacheProviderReg.Match(output.Stderr.Bytes()) { - return nil, nil - } + if err != nil && httpStatusCacheProviderReg.Match(output.Stderr.Bytes()) { + return nil, nil + } - if err := errWriter.Flush(); err != nil { - return nil, err - } + if err := errWriter.Flush(); err != nil { + return nil, err } return output, err diff --git a/pkg/log/format/formatter.go b/pkg/log/format/formatter.go index 09a0691e6..f0bf612a9 100644 --- a/pkg/log/format/formatter.go +++ b/pkg/log/format/formatter.go @@ -2,6 +2,7 @@ package format import ( "bytes" + "sync" "github.com/gruntwork-io/terragrunt/internal/errors" "github.com/gruntwork-io/terragrunt/pkg/log" @@ -17,6 +18,7 @@ type Formatter struct { placeholders placeholders.Placeholders disableColors bool relativePather *options.RelativePather + mu sync.Mutex } // NewFormatter returns a new Formatter instance with default values. @@ -28,6 +30,9 @@ func NewFormatter(phs placeholders.Placeholders) *Formatter { // Format implements logrus.Format. func (formatter *Formatter) Format(entry *log.Entry) ([]byte, error) { + formatter.mu.Lock() + defer formatter.mu.Unlock() + if formatter.placeholders == nil { return nil, nil } diff --git a/pkg/log/writer/writer.go b/pkg/log/writer/writer.go index 8dd241a8e..f349681f7 100644 --- a/pkg/log/writer/writer.go +++ b/pkg/log/writer/writer.go @@ -3,7 +3,6 @@ package writer import ( "strings" - "sync" "time" "github.com/gruntwork-io/terragrunt/pkg/log" @@ -18,7 +17,6 @@ type Writer struct { defaultLevel log.Level msgSeparator string parseFunc WriterParseFunc - mu sync.Mutex } // New returns a new Writer instance with fields assigned to default values. @@ -42,9 +40,6 @@ func (writer *Writer) SetOption(opts ...Option) { // Write implements `io.Writer` interface. func (writer *Writer) Write(p []byte) (n int, err error) { - writer.mu.Lock() - defer writer.mu.Unlock() - var ( str = string(p) strs = []string{str} From fa2ed61009e8a872338376716d650892aa32bc1d Mon Sep 17 00:00:00 2001 From: Levko Burburas Date: Wed, 27 Nov 2024 01:31:29 +0200 Subject: [PATCH 14/21] chore: fix test --- configstack/module.go | 2 +- pkg/log/format/formatter.go | 6 +++--- shell/git.go | 2 ++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/configstack/module.go b/configstack/module.go index e104e3ed4..4ecccf0ae 100644 --- a/configstack/module.go +++ b/configstack/module.go @@ -256,7 +256,7 @@ func FindWhereWorkingDirIsIncluded(ctx context.Context, opts *options.Terragrunt cfgOptions.OriginalTerragruntConfigPath = opts.OriginalTerragruntConfigPath cfgOptions.TerraformCommand = opts.TerraformCommand cfgOptions.NonInteractive = true - cfgOptions.Logger.SetOptions(log.WithHooks(NewForceLogLevelHook(log.DebugLevel))) + cfgOptions.Logger = opts.Logger.WithOptions(log.WithHooks(NewForceLogLevelHook(log.DebugLevel))) // build stack from config directory stack, err := FindStackInSubfolders(ctx, cfgOptions, WithChildTerragruntConfig(terragruntConfig)) diff --git a/pkg/log/format/formatter.go b/pkg/log/format/formatter.go index f0bf612a9..73af5288f 100644 --- a/pkg/log/format/formatter.go +++ b/pkg/log/format/formatter.go @@ -30,9 +30,6 @@ func NewFormatter(phs placeholders.Placeholders) *Formatter { // Format implements logrus.Format. func (formatter *Formatter) Format(entry *log.Entry) ([]byte, error) { - formatter.mu.Lock() - defer formatter.mu.Unlock() - if formatter.placeholders == nil { return nil, nil } @@ -52,6 +49,9 @@ func (formatter *Formatter) Format(entry *log.Entry) ([]byte, error) { return nil, err } + formatter.mu.Lock() + defer formatter.mu.Unlock() + if str != "" { if _, err := buf.WriteString(str); err != nil { return nil, errors.New(err) diff --git a/shell/git.go b/shell/git.go index 575778fee..02cfae6ec 100644 --- a/shell/git.go +++ b/shell/git.go @@ -36,6 +36,7 @@ func GitTopLevelDir(ctx context.Context, terragruntOptions *options.TerragruntOp return "", err } + opts.Logger = terragruntOptions.Logger.Clone() opts.Env = terragruntOptions.Env opts.Writer = &stdout opts.ErrWriter = &stderr @@ -66,6 +67,7 @@ func GitRepoTags(ctx context.Context, opts *options.TerragruntOptions, gitRepo * return nil, err } + gitOpts.Logger = opts.Logger.Clone() gitOpts.Env = opts.Env gitOpts.Writer = &stdout gitOpts.ErrWriter = &stderr From 696828da77202e0ed3015028d9de372859b9dd74 Mon Sep 17 00:00:00 2001 From: Levko Burburas Date: Wed, 27 Nov 2024 14:55:57 +0200 Subject: [PATCH 15/21] chore: fix test --- cli/provider_cache.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cli/provider_cache.go b/cli/provider_cache.go index 33648d0a7..9e7e3b834 100644 --- a/cli/provider_cache.go +++ b/cli/provider_cache.go @@ -353,12 +353,14 @@ func runTerraformCommand(ctx context.Context, opts *options.TerragruntOptions, a output, err := shell.RunTerraformCommandWithOutput(ctx, cloneOpts, cloneOpts.TerraformCliArgs...) // If the Terraform error matches `httpStatusCacheProviderReg` we ignore it and hide the log from users, otherwise we process the error as is. - if err != nil && httpStatusCacheProviderReg.Match(output.Stderr.Bytes()) { - return nil, nil - } + if err != nil { + if httpStatusCacheProviderReg.Match(output.Stderr.Bytes()) { + return nil, nil + } - if err := errWriter.Flush(); err != nil { - return nil, err + if err := errWriter.Flush(); err != nil { + return nil, err + } } return output, err From 417e58e351e44bb4c561714ce9e30d03583352af Mon Sep 17 00:00:00 2001 From: Levko Burburas Date: Wed, 27 Nov 2024 15:27:08 +0200 Subject: [PATCH 16/21] fix: cache provider --- cli/provider_cache.go | 15 +++++++-------- util/trap_writer.go | 5 ++++- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/cli/provider_cache.go b/cli/provider_cache.go index 9e7e3b834..9112f6e79 100644 --- a/cli/provider_cache.go +++ b/cli/provider_cache.go @@ -336,7 +336,8 @@ func runTerraformCommand(ctx context.Context, opts *options.TerragruntOptions, a errWriter := util.NewTrapWriter(opts.ErrWriter) // add -no-color flag to args if it was set in Terragrunt arguments - if util.ListContainsElement(opts.TerraformCliArgs, terraform.FlagNameNoColor) { + if util.ListContainsElement(opts.TerraformCliArgs, terraform.FlagNameNoColor) && + !util.ListContainsElement(args, terraform.FlagNameNoColor) { args = append(args, terraform.FlagNameNoColor) } @@ -353,14 +354,12 @@ func runTerraformCommand(ctx context.Context, opts *options.TerragruntOptions, a output, err := shell.RunTerraformCommandWithOutput(ctx, cloneOpts, cloneOpts.TerraformCliArgs...) // If the Terraform error matches `httpStatusCacheProviderReg` we ignore it and hide the log from users, otherwise we process the error as is. - if err != nil { - if httpStatusCacheProviderReg.Match(output.Stderr.Bytes()) { - return nil, nil - } + if err != nil && httpStatusCacheProviderReg.Match(output.Stderr.Bytes()) { + return nil, nil + } - if err := errWriter.Flush(); err != nil { - return nil, err - } + if err := errWriter.Flush(); err != nil { + return nil, err } return output, err diff --git a/util/trap_writer.go b/util/trap_writer.go index c6c89d469..81acacdd6 100644 --- a/util/trap_writer.go +++ b/util/trap_writer.go @@ -32,7 +32,10 @@ func (trap *TrapWriter) Flush() error { } // Write implements `io.Writer` interface. -func (trap *TrapWriter) Write(msg []byte) (int, error) { +func (trap *TrapWriter) Write(d []byte) (int, error) { + msg := make([]byte, len(d)) + copy(msg, d) + trap.msgs = append(trap.msgs, msg) return len(msg), nil From ace8c2695d4bf539617fad4a9c7201b47bfa9af6 Mon Sep 17 00:00:00 2001 From: Levko Burburas Date: Wed, 27 Nov 2024 15:32:57 +0200 Subject: [PATCH 17/21] chore: update docs --- docs/_docs/04_reference/cli-options.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/_docs/04_reference/cli-options.md b/docs/_docs/04_reference/cli-options.md index 1c132977c..6171712e0 100644 --- a/docs/_docs/04_reference/cli-options.md +++ b/docs/_docs/04_reference/cli-options.md @@ -1483,9 +1483,10 @@ DEPRECATED: Use [terragrunt-log-format](#terragrunt-log-format). When this flag is set, Terragrunt will output its logs in JSON format. -### terragrunt-tf-logs-to-json +# terragrunt-tf-logs-to-json DEPRECATED: Use [terragrunt-log-format](#terragrunt-log-format). OpenTofu/Terraform `stdout` and `stderr` is wrapped in JSON by default with `--terragurnt-log-format json` flag if `--terragrunt-forward-tf-stdout` flag is not specified. +In other words, the previous behavior with the `--terragrunt-json-log --terragrunt-tf-logs-to-json` flags is now equivalent to `--terragrunt-log-format json` and the previous behavior with the `--terragrunt-json-log` is now equivalent to `--terragrunt-log-format json --terragrunt-forward-tf-stdout`. **CLI Arg**: `--terragrunt-tf-logs-to-json`
**Environment Variable**: `TERRAGRUNT_TF_JSON_LOG` (set to `true`)
From d4432c9b92aeae507540afc24e4c4259e7734791 Mon Sep 17 00:00:00 2001 From: Levko Burburas Date: Wed, 27 Nov 2024 15:44:50 +0200 Subject: [PATCH 18/21] chore: fix strict lint --- cli/provider_cache.go | 2 +- internal/strict/strict.go | 22 +++++++++++----------- pkg/log/format/options/escape.go | 4 ++-- pkg/log/format/options/path_format.go | 26 +++++++++++++++----------- 4 files changed, 29 insertions(+), 25 deletions(-) diff --git a/cli/provider_cache.go b/cli/provider_cache.go index 9112f6e79..a1969f881 100644 --- a/cli/provider_cache.go +++ b/cli/provider_cache.go @@ -355,7 +355,7 @@ func runTerraformCommand(ctx context.Context, opts *options.TerragruntOptions, a output, err := shell.RunTerraformCommandWithOutput(ctx, cloneOpts, cloneOpts.TerraformCliArgs...) // If the Terraform error matches `httpStatusCacheProviderReg` we ignore it and hide the log from users, otherwise we process the error as is. if err != nil && httpStatusCacheProviderReg.Match(output.Stderr.Bytes()) { - return nil, nil + return new(util.CmdOutput), nil } if err := errWriter.Flush(); err != nil { diff --git a/internal/strict/strict.go b/internal/strict/strict.go index a59328310..3d2622461 100644 --- a/internal/strict/strict.go +++ b/internal/strict/strict.go @@ -85,47 +85,47 @@ type Controls map[string]Control //nolint:lll,gochecknoglobals,stylecheck var StrictControls = Controls{ SpinUp: { - Error: errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all apply` instead.", SpinUp), + Error: errors.Errorf("the `%s` command is no longer supported. Use `terragrunt run-all apply` instead", SpinUp), Warning: fmt.Sprintf("The `%s` command is deprecated and will be removed in a future version. Use `terragrunt run-all apply` instead.", SpinUp), }, TearDown: { - Error: errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all destroy` instead.", TearDown), + Error: errors.Errorf("the `%s` command is no longer supported. Use `terragrunt run-all destroy` instead", TearDown), Warning: fmt.Sprintf("The `%s` command is deprecated and will be removed in a future version. Use `terragrunt run-all destroy` instead.", TearDown), }, PlanAll: { - Error: errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all plan` instead.", PlanAll), + Error: errors.Errorf("the `%s` command is no longer supported. Use `terragrunt run-all plan` instead", PlanAll), Warning: fmt.Sprintf("The `%s` command is deprecated and will be removed in a future version. Use `terragrunt run-all plan` instead.", PlanAll), }, ApplyAll: { - Error: errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all apply` instead.", ApplyAll), + Error: errors.Errorf("the `%s` command is no longer supported. Use `terragrunt run-all apply` instead", ApplyAll), Warning: fmt.Sprintf("The `%s` command is deprecated and will be removed in a future version. Use `terragrunt run-all apply` instead.", ApplyAll), }, DestroyAll: { - Error: errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all destroy` instead.", DestroyAll), + Error: errors.Errorf("the `%s` command is no longer supported. Use `terragrunt run-all destroy` instead", DestroyAll), Warning: fmt.Sprintf("The `%s` command is deprecated and will be removed in a future version. Use `terragrunt run-all destroy` instead.", DestroyAll), }, OutputAll: { - Error: errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all output` instead.", OutputAll), + Error: errors.Errorf("the `%s` command is no longer supported. Use `terragrunt run-all output` instead", OutputAll), Warning: fmt.Sprintf("The `%s` command is deprecated and will be removed in a future version. Use `terragrunt run-all output` instead.", OutputAll), }, ValidateAll: { - Error: errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all validate` instead.", ValidateAll), + Error: errors.Errorf("the `%s` command is no longer supported. Use `terragrunt run-all validate` instead", ValidateAll), Warning: fmt.Sprintf("The `%s` command is deprecated and will be removed in a future version. Use `terragrunt run-all validate` instead.", ValidateAll), }, SkipDependenciesInputs: { - Error: errors.Errorf("The `%s` option is deprecated. Reading inputs from dependencies has been deprecated and will be removed in a future version of Terragrunt. To continue using inputs from dependencies, forward them as outputs.", SkipDependenciesInputs), + Error: errors.Errorf("the `%s` option is deprecated. Reading inputs from dependencies has been deprecated and will be removed in a future version of Terragrunt. To continue using inputs from dependencies, forward them as outputs", SkipDependenciesInputs), Warning: fmt.Sprintf("The `%s` option is deprecated and will be removed in a future version of Terragrunt. Reading inputs from dependencies has been deprecated. To continue using inputs from dependencies, forward them as outputs.", SkipDependenciesInputs), }, DisableLogFormatting: { - Error: errors.Errorf("The `--%s` flag is no longer supported. Use `--terragrunt-log-format=key-value` instead.", DisableLogFormatting), + Error: errors.Errorf("the `--%s` flag is no longer supported. Use `--terragrunt-log-format=key-value` instead", DisableLogFormatting), Warning: fmt.Sprintf("The `--%s` flag is deprecated and will be removed in a future version. Use `--terragrunt-log-format=key-value` instead.", DisableLogFormatting), }, JSONLog: { - Error: errors.Errorf("The `--%s` flag is no longer supported. Use `--terragrunt-log-format=json` instead.", JSONLog), + Error: errors.Errorf("the `--%s` flag is no longer supported. Use `--terragrunt-log-format=json` instead", JSONLog), Warning: fmt.Sprintf("The `--%s` flag is deprecated and will be removed in a future version. Use `--terragrunt-log-format=json` instead.", JSONLog), }, TfLogJSON: { - Error: errors.Errorf("The `--%s` flag is no longer supported. Use `--terragrunt-log-format=json` instead.", TfLogJSON), + Error: errors.Errorf("the `--%s` flag is no longer supported. Use `--terragrunt-log-format=json` instead", TfLogJSON), Warning: fmt.Sprintf("The `--%s` flag is deprecated and will be removed in a future version. Use `--terragrunt-log-format=json` instead.", TfLogJSON), }, } diff --git a/pkg/log/format/options/escape.go b/pkg/log/format/options/escape.go index 745116c23..80edd56f1 100644 --- a/pkg/log/format/options/escape.go +++ b/pkg/log/format/options/escape.go @@ -30,13 +30,13 @@ func (option *EscapeOption) Format(_ *Data, val any) (any, error) { return val, nil } - b, err := json.Marshal(val) + jsonStr, err := json.Marshal(val) if err != nil { return "", errors.New(err) } // Trim the beginning and trailing " character. - return string(b[1 : len(b)-1]), nil + return string(jsonStr[1 : len(jsonStr)-1]), nil } // Escape creates the option to escape text. diff --git a/pkg/log/format/options/path_format.go b/pkg/log/format/options/path_format.go index 1aa4f426c..168b25b46 100644 --- a/pkg/log/format/options/path_format.go +++ b/pkg/log/format/options/path_format.go @@ -49,21 +49,11 @@ func (option *PathFormatOption) Format(data *Data, val any) (any, error) { return data.RelativePather.ReplaceAbsPaths(str), nil case ShortRelativePath: - if str == data.BaseDir { - return "", nil - } - if data.RelativePather == nil { break } - str = data.RelativePather.ReplaceAbsPaths(str) - - if strings.HasPrefix(str, log.CurDirWithSeparator) { - return str[len(log.CurDirWithSeparator):], nil - } - - return str, nil + return option.shortRelativePath(data, str), nil case ShortPath: if str == data.BaseDir { return "", nil @@ -80,6 +70,20 @@ func (option *PathFormatOption) Format(data *Data, val any) (any, error) { return val, nil } +func (option *PathFormatOption) shortRelativePath(data *Data, str string) string { + if str == data.BaseDir { + return "" + } + + str = data.RelativePather.ReplaceAbsPaths(str) + + if strings.HasPrefix(str, log.CurDirWithSeparator) { + return str[len(log.CurDirWithSeparator):] + } + + return str +} + // PathFormat creates the option to format the paths. func PathFormat(val PathFormatValue, allowed ...PathFormatValue) Option { list := pathFormatList From ae33366a7a09bd62fdcc9ad11a491831f5677e25 Mon Sep 17 00:00:00 2001 From: Levko Burburas Date: Wed, 27 Nov 2024 15:52:40 +0200 Subject: [PATCH 19/21] fix: markdownlint --- docs/_docs/04_reference/cli-options.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_docs/04_reference/cli-options.md b/docs/_docs/04_reference/cli-options.md index 6171712e0..9ac672e64 100644 --- a/docs/_docs/04_reference/cli-options.md +++ b/docs/_docs/04_reference/cli-options.md @@ -1483,7 +1483,7 @@ DEPRECATED: Use [terragrunt-log-format](#terragrunt-log-format). When this flag is set, Terragrunt will output its logs in JSON format. -# terragrunt-tf-logs-to-json +### terragrunt-tf-logs-to-json DEPRECATED: Use [terragrunt-log-format](#terragrunt-log-format). OpenTofu/Terraform `stdout` and `stderr` is wrapped in JSON by default with `--terragurnt-log-format json` flag if `--terragrunt-forward-tf-stdout` flag is not specified. In other words, the previous behavior with the `--terragrunt-json-log --terragrunt-tf-logs-to-json` flags is now equivalent to `--terragrunt-log-format json` and the previous behavior with the `--terragrunt-json-log` is now equivalent to `--terragrunt-log-format json --terragrunt-forward-tf-stdout`. From 851237417a966313d86a5670332f9b47cc64e499 Mon Sep 17 00:00:00 2001 From: Levko Burburas Date: Wed, 27 Nov 2024 16:08:40 +0200 Subject: [PATCH 20/21] chore: fix punctuations --- docs/_docs/02_features/custom-log-format.md | 8 +++++--- internal/strict/strict.go | 22 ++++++++++----------- pkg/log/format/format.go | 8 ++++---- shell/run_shell_cmd.go | 2 +- test/integration_test.go | 2 +- 5 files changed, 22 insertions(+), 20 deletions(-) diff --git a/docs/_docs/02_features/custom-log-format.md b/docs/_docs/02_features/custom-log-format.md index 3546a3790..52ffc4a1e 100644 --- a/docs/_docs/02_features/custom-log-format.md +++ b/docs/_docs/02_features/custom-log-format.md @@ -56,9 +56,11 @@ Placeholders have preset names: * `%prefix` - Path to the working directory were Terragrunt is running. +* `%msg` - Log message. + * `%tf-path` - Path to the OpenTofu/Terraform executable (as defined by [terragrunt-tfpath](https://terragrunt.gruntwork.io/docs/reference/cli-options/#terragrunt-tfpath)). -* `%msg` - Log message. +* `%tf-command-args` - Arguments of the executed OpenTofu/Terraform command. * `%t` - Tab. @@ -289,11 +291,11 @@ The examples below replicate the preset formats specified with `--terragrunt-log `--terragrunt-log-format key-value` ```shell ---terragrunt-log-custom-format "time=%time(format=rfc3339) level=%level prefix=%prefix(path=short-relative) tfpath=%tf-path(path=filename) msg=%msg(path=relative,color=disable)" +--terragrunt-log-custom-format "time=%time(format=rfc3339) level=%level prefix=%prefix(path=short-relative) tf-path=%tf-path(path=filename) msg=%msg(path=relative,color=disable)" ``` `--terragrunt-log-format json` ```shell ---terragrunt-log-custom-format '{"time":"%time(format=rfc3339,escape=json)", "level":"%level(escape=json)", "prefix":"%prefix(path=short-relative,escape=json)", "tfpath":"%tf-path(path=filename,escape=json)", "msg":"%msg(path=relative,escape=json,color=disable)"}' +--terragrunt-log-custom-format '{"time":"%time(format=rfc3339,escape=json)", "level":"%level(escape=json)", "prefix":"%prefix(path=short-relative,escape=json)", "tf-path":"%tf-path(path=filename,escape=json)", "msg":"%msg(path=relative,escape=json,color=disable)"}' ``` diff --git a/internal/strict/strict.go b/internal/strict/strict.go index 3d2622461..dbfa26bcf 100644 --- a/internal/strict/strict.go +++ b/internal/strict/strict.go @@ -85,47 +85,47 @@ type Controls map[string]Control //nolint:lll,gochecknoglobals,stylecheck var StrictControls = Controls{ SpinUp: { - Error: errors.Errorf("the `%s` command is no longer supported. Use `terragrunt run-all apply` instead", SpinUp), + Error: errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all apply` instead.", SpinUp), //nolint:revive Warning: fmt.Sprintf("The `%s` command is deprecated and will be removed in a future version. Use `terragrunt run-all apply` instead.", SpinUp), }, TearDown: { - Error: errors.Errorf("the `%s` command is no longer supported. Use `terragrunt run-all destroy` instead", TearDown), + Error: errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all destroy` instead.", TearDown), //nolint:revive Warning: fmt.Sprintf("The `%s` command is deprecated and will be removed in a future version. Use `terragrunt run-all destroy` instead.", TearDown), }, PlanAll: { - Error: errors.Errorf("the `%s` command is no longer supported. Use `terragrunt run-all plan` instead", PlanAll), + Error: errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all plan` instead.", PlanAll), //nolint:revive Warning: fmt.Sprintf("The `%s` command is deprecated and will be removed in a future version. Use `terragrunt run-all plan` instead.", PlanAll), }, ApplyAll: { - Error: errors.Errorf("the `%s` command is no longer supported. Use `terragrunt run-all apply` instead", ApplyAll), + Error: errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all apply` instead.", ApplyAll), //nolint:revive Warning: fmt.Sprintf("The `%s` command is deprecated and will be removed in a future version. Use `terragrunt run-all apply` instead.", ApplyAll), }, DestroyAll: { - Error: errors.Errorf("the `%s` command is no longer supported. Use `terragrunt run-all destroy` instead", DestroyAll), + Error: errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all destroy` instead.", DestroyAll), //nolint:revive Warning: fmt.Sprintf("The `%s` command is deprecated and will be removed in a future version. Use `terragrunt run-all destroy` instead.", DestroyAll), }, OutputAll: { - Error: errors.Errorf("the `%s` command is no longer supported. Use `terragrunt run-all output` instead", OutputAll), + Error: errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all output` instead.", OutputAll), //nolint:revive Warning: fmt.Sprintf("The `%s` command is deprecated and will be removed in a future version. Use `terragrunt run-all output` instead.", OutputAll), }, ValidateAll: { - Error: errors.Errorf("the `%s` command is no longer supported. Use `terragrunt run-all validate` instead", ValidateAll), + Error: errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all validate` instead.", ValidateAll), //nolint:revive Warning: fmt.Sprintf("The `%s` command is deprecated and will be removed in a future version. Use `terragrunt run-all validate` instead.", ValidateAll), }, SkipDependenciesInputs: { - Error: errors.Errorf("the `%s` option is deprecated. Reading inputs from dependencies has been deprecated and will be removed in a future version of Terragrunt. To continue using inputs from dependencies, forward them as outputs", SkipDependenciesInputs), + Error: errors.Errorf("The `%s` option is deprecated. Reading inputs from dependencies has been deprecated and will be removed in a future version of Terragrunt. To continue using inputs from dependencies, forward them as outputs.", SkipDependenciesInputs), //nolint:revive Warning: fmt.Sprintf("The `%s` option is deprecated and will be removed in a future version of Terragrunt. Reading inputs from dependencies has been deprecated. To continue using inputs from dependencies, forward them as outputs.", SkipDependenciesInputs), }, DisableLogFormatting: { - Error: errors.Errorf("the `--%s` flag is no longer supported. Use `--terragrunt-log-format=key-value` instead", DisableLogFormatting), + Error: errors.Errorf("The `--%s` flag is no longer supported. Use `--terragrunt-log-format=key-value` instead.", DisableLogFormatting), //nolint:revive Warning: fmt.Sprintf("The `--%s` flag is deprecated and will be removed in a future version. Use `--terragrunt-log-format=key-value` instead.", DisableLogFormatting), }, JSONLog: { - Error: errors.Errorf("the `--%s` flag is no longer supported. Use `--terragrunt-log-format=json` instead", JSONLog), + Error: errors.Errorf("The `--%s` flag is no longer supported. Use `--terragrunt-log-format=json` instead.", JSONLog), //nolint:revive Warning: fmt.Sprintf("The `--%s` flag is deprecated and will be removed in a future version. Use `--terragrunt-log-format=json` instead.", JSONLog), }, TfLogJSON: { - Error: errors.Errorf("the `--%s` flag is no longer supported. Use `--terragrunt-log-format=json` instead", TfLogJSON), + Error: errors.Errorf("The `--%s` flag is no longer supported. Use `--terragrunt-log-format=json` instead.", TfLogJSON), //nolint:revive Warning: fmt.Sprintf("The `--%s` flag is deprecated and will be removed in a future version. Use `--terragrunt-log-format=json` instead.", TfLogJSON), }, } diff --git a/pkg/log/format/format.go b/pkg/log/format/format.go index 8fdaa0f3f..9adf9c252 100644 --- a/pkg/log/format/format.go +++ b/pkg/log/format/format.go @@ -83,18 +83,18 @@ func NewJSONFormat() Placeholders { Escape(JSONEscape), ), Field(WorkDirKeyName, - Prefix(`, "workingDir":"`), + Prefix(`, "working-dir":"`), Suffix(`"`), Escape(JSONEscape), ), Field(TFPathKeyName, - Prefix(`, "tfpath":"`), + Prefix(`, "tf-path":"`), Suffix(`"`), PathFormat(FilenamePath), Escape(JSONEscape), ), Field(TFCmdArgsKeyName, - Prefix(`, "executedCommandArgs":[`), + Prefix(`, "tf-command-args":[`), Suffix(`]`), Escape(JSONEscape), ), @@ -123,7 +123,7 @@ func NewKeyValueFormat() Placeholders { PathFormat(ShortRelativePath), ), Field(TFPathKeyName, - Prefix(" tfpath="), + Prefix(" tf-path="), PathFormat(FilenamePath), ), Message( diff --git a/shell/run_shell_cmd.go b/shell/run_shell_cmd.go index cee2a9372..2b87a1d3e 100644 --- a/shell/run_shell_cmd.go +++ b/shell/run_shell_cmd.go @@ -138,7 +138,7 @@ func RunShellCommandWithOutput( WithField(placeholders.TFPathKeyName, filepath.Base(opts.TerraformPath)). WithField(placeholders.TFCmdArgsKeyName, args) - if opts.JSONLogFormat { + if opts.JSONLogFormat && !cli.Args(args).Normalize(cli.SingleDashFlag).Contains(terraform.FlagNameJSON) { outWriter = writer.New( writer.WithLogger(logger.WithOptions(log.WithOutput(errWriter))), writer.WithDefaultLevel(log.StdoutLevel), diff --git a/test/integration_test.go b/test/integration_test.go index 5cd20a110..29291aeec 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -375,7 +375,7 @@ func TestLogFormatKeyValueOutput(t *testing.T) { require.NoError(t, err) for _, prefixName := range []string{"app", "dep"} { - assert.Contains(t, stderr, "level=stdout prefix="+prefixName+" tfpath="+wrappedBinary()+" msg=Initializing provider plugins...\n") + assert.Contains(t, stderr, "level=stdout prefix="+prefixName+" tf-path="+wrappedBinary()+" msg=Initializing provider plugins...\n") assert.Contains(t, stderr, "level=debug prefix="+prefixName+" msg=Reading Terragrunt config file at ./"+prefixName+"/terragrunt.hcl\n") } }) From 2c054786191649f39b0e835d98a3d4bf6cd61e84 Mon Sep 17 00:00:00 2001 From: Levko Burburas Date: Wed, 27 Nov 2024 16:27:01 +0200 Subject: [PATCH 21/21] chore: fix test --- shell/run_shell_cmd_output_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell/run_shell_cmd_output_test.go b/shell/run_shell_cmd_output_test.go index 37a249a12..c3ce3fcf8 100644 --- a/shell/run_shell_cmd_output_test.go +++ b/shell/run_shell_cmd_output_test.go @@ -65,7 +65,7 @@ func TestCommandOutputPrefix(t *testing.T) { terraformPath := "testdata/test_outputs.sh" prefixedOutput := []string{} for _, line := range FullOutput { - prefixedOutput = append(prefixedOutput, fmt.Sprintf("prefix=%s tfpath=%s msg=%s", prefix, filepath.Base(terraformPath), line)) + prefixedOutput = append(prefixedOutput, fmt.Sprintf("prefix=%s tf-path=%s msg=%s", prefix, filepath.Base(terraformPath), line)) } logFormatter := format.NewFormatter(format.NewKeyValueFormat())