Skip to content

Commit

Permalink
Fixing output STDOUT logs to stdout (#3409)
Browse files Browse the repository at this point in the history 
* fix: output log to stdout

* chore: fix tests

* chore: fix tests

* chore: fix tests

* chore: fix grammar

Co-authored-by: Yousif Akbar <[email protected]>

---------

Co-authored-by: Yousif Akbar <[email protected]>
  • Loading branch information
@levkohimins @yhakbar
levkohimins and yhakbar authored Sep 16, 2024
1 parent 839846c commit 64baa28
Show file tree
Hide file tree
Showing 11 changed files with 111 additions and 80 deletions.
6 changes: 3 additions & 3 deletions pkg/cli/args.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import (
"strings"
)

const tailMinArgsLen = 2

const (
SingleDashFlag NormalizeActsType = iota
DoubleDashFlag
Expand Down Expand Up @@ -42,9 +44,7 @@ func (args Args) Last() string {
// Tail returns the rest of the arguments (not the first one)
// or else an empty string slice
func (args Args) Tail() Args {
const minArgsLen = 2

if args.Len() < minArgsLen {
if args.Len() < tailMinArgsLen {
return []string{}
}

Expand Down
19 changes: 16 additions & 3 deletions shell/run_shell_cmd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ func RunShellCommandWithOutput(
logger := opts.Logger.WithField(format.TFBinaryKeyName, filepath.Base(opts.TerraformPath))

outWriter = writer.New(
writer.WithLogger(logger.WithOptions(log.WithOutput(outWriter))),
writer.WithLogger(logger.WithOptions(log.WithOutput(errWriter))),
writer.WithDefaultLevel(log.StdoutLevel),
writer.WithMsgSeparator(logMsgSeparator),
)
Expand All @@ -162,7 +162,7 @@ func RunShellCommandWithOutput(
writer.WithLogger(logger.WithOptions(log.WithOutput(errWriter))),
writer.WithDefaultLevel(log.StderrLevel),
writer.WithMsgSeparator(logMsgSeparator),
writer.WithParseFunc(terraform.ParseLogFunc(tfLogMsgPrefix)),
writer.WithParseFunc(terraform.ParseLogFunc(tfLogMsgPrefix, false)),
)
}
}
Expand Down Expand Up @@ -453,5 +453,18 @@ func shouldForceForwardTFStdout(args cli.Args) bool {
terraform.CommandNameConsole,
}

return collections.ListContainsElement(tfCommands, args.CommandName()) || args.Tail().Contains(terraform.FlagNameJSON)
tfFlags := []string{
terraform.FlagNameJSON,
terraform.FlagNameVersion,
terraform.FlagNameHelpLong,
terraform.FlagNameHelpShort,
}

for _, flag := range tfFlags {
if args.Normalize(cli.SingleDashFlag).Contains(flag) {
return true
}
}

return collections.ListContainsElement(tfCommands, args.CommandName())
}
47 changes: 30 additions & 17 deletions terraform/log.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,13 @@ import (
"strings"
"time"

"github.com/gruntwork-io/go-commons/errors"
"github.com/gruntwork-io/terragrunt/pkg/log"
"github.com/gruntwork-io/terragrunt/pkg/log/writer"
)

const parseLogNumberOfValues = 4

var (
// logTimestampFormat is TF_LOG timestamp formats.
logTimestampFormat = "2006-01-02T15:04:05.000Z0700"
Expand All @@ -22,41 +25,51 @@ var (
tfLogTimeLevelMsgReg = regexp.MustCompile(`(?i)(^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}\S*)\s*\[(trace|debug|warn|info|error)\]\s*(.+\S)$`)
)

func ParseLogFunc(msgPrefix string) writer.WriterParseFunc {
// ParseLogFunc wraps `ParseLog` to add msg prefix and bypasses the the parse erorr if `returnError` is false,
// since returning the error for `log/writer` will cause TG to fall with a `broken pipe` error.
func ParseLogFunc(msgPrefix string, returnError bool) writer.WriterParseFunc {
return func(str string) (msg string, ptrTime *time.Time, ptrLevel *log.Level, err error) {
return ParseLog(msgPrefix, str)
if msg, ptrTime, ptrLevel, err = ParseLog(str); err != nil {
if returnError {
return str, nil, nil, err
}

return str, nil, nil, nil
}

return msgPrefix + msg, ptrTime, ptrLevel, nil
}
}

func ParseLog(msgPrefix, str string) (msg string, ptrTime *time.Time, ptrLevel *log.Level, err error) {
const numberOfValues = 4

func ParseLog(str string) (msg string, ptrTime *time.Time, ptrLevel *log.Level, err error) {
if !tfLogTimeLevelMsgReg.MatchString(str) {
return str, nil, nil, nil
return str, nil, nil, errors.Errorf("could not parse string %q: does not match a known format", str)
}

match := tfLogTimeLevelMsgReg.FindStringSubmatch(str)
if len(match) != numberOfValues {
return str, nil, nil, nil
if len(match) != parseLogNumberOfValues {
return str, nil, nil, errors.Errorf("could not parse string %q: does not match a known format", str)
}

timeStr, levelStr, msg := match[1], match[2], match[3]

if levelStr != "" {
if level, err := log.ParseLevel(strings.ToLower(levelStr)); err == nil {
ptrLevel = &level
} else {
msg = "[" + levelStr + "] " + msg
level, err := log.ParseLevel(strings.ToLower(levelStr))
if err != nil {
return str, nil, nil, errors.Errorf("could not parse level %q: %w", levelStr, err)
}

ptrLevel = &level
}

if timeStr != "" {
if time, err := time.Parse(logTimestampFormat, timeStr); err == nil {
ptrTime = &time
} else {
msg = timeStr + " " + msg
time, err := time.Parse(logTimestampFormat, timeStr)
if err != nil {
return str, nil, nil, errors.Errorf("could not parse time %q: %w", timeStr, err)
}

ptrTime = &time
}

return msgPrefix + msg, ptrTime, ptrLevel, nil
return msg, ptrTime, ptrLevel, nil
}
7 changes: 5 additions & 2 deletions terraform/terraform.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,11 @@ const (
CommandNameShow = "show"
CommandNameVersion = "version"

FlagNameJSON = "-json"
FlagNameNoColor = "-no-color"
FlagNameHelpLong = "-help"
FlagNameHelpShort = "-h"
FlagNameVersion = "-version"
FlagNameJSON = "-json"
FlagNameNoColor = "-no-color"
// `apply -destroy` is alias for `destroy`
FlagNameDestroy = "-destroy"

Expand Down
12 changes: 6 additions & 6 deletions test/integration_auto_retry_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ func TestAutoRetryBasicRerun(t *testing.T) {
out := new(bytes.Buffer)
rootPath := copyEnvironment(t, testFixtureAutoRetryRerun)
modulePath := util.JoinPath(rootPath, testFixtureAutoRetryRerun)
err := runTerragruntCommand(t, "terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir "+modulePath, out, os.Stderr)
err := runTerragruntCommand(t, "terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-forward-tf-stdout --terragrunt-working-dir "+modulePath, out, os.Stderr)

require.NoError(t, err)
assert.Contains(t, out.String(), "Apply complete!")
Expand All @@ -55,7 +55,7 @@ func TestAutoRetryExhaustRetries(t *testing.T) {
out := new(bytes.Buffer)
rootPath := copyEnvironment(t, testFixtureAutoRetryExhaust)
modulePath := util.JoinPath(rootPath, testFixtureAutoRetryExhaust)
err := runTerragruntCommand(t, "terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir "+modulePath, out, os.Stderr)
err := runTerragruntCommand(t, "terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-forward-tf-stdout --terragrunt-working-dir "+modulePath, out, os.Stderr)

require.Error(t, err)
assert.Contains(t, out.String(), "Failed to load backend")
Expand All @@ -68,7 +68,7 @@ func TestAutoRetryCustomRetryableErrors(t *testing.T) {
out := new(bytes.Buffer)
rootPath := copyEnvironment(t, testFixtureAutoRetryCustomErrors)
modulePath := util.JoinPath(rootPath, testFixtureAutoRetryCustomErrors)
err := runTerragruntCommand(t, "terragrunt apply --auto-approve --terragrunt-non-interactive --terragrunt-working-dir "+modulePath, out, os.Stderr)
err := runTerragruntCommand(t, "terragrunt apply --auto-approve --terragrunt-non-interactive --terragrunt-forward-tf-stdout --terragrunt-working-dir "+modulePath, out, os.Stderr)

require.NoError(t, err)
assert.Contains(t, out.String(), "My own little error")
Expand Down Expand Up @@ -101,7 +101,7 @@ func TestAutoRetryCustomRetryableErrorsFailsWhenRetryableErrorsNotSet(t *testing
out := new(bytes.Buffer)
rootPath := copyEnvironment(t, testFixtureAutoRetryCustomErrorsNotSet)
modulePath := util.JoinPath(rootPath, testFixtureAutoRetryCustomErrorsNotSet)
err := runTerragruntCommand(t, "terragrunt apply --auto-approve --terragrunt-non-interactive --terragrunt-working-dir "+modulePath, out, os.Stderr)
err := runTerragruntCommand(t, "terragrunt apply --auto-approve --terragrunt-non-interactive --terragrunt-forward-tf-stdout --terragrunt-working-dir "+modulePath, out, os.Stderr)

require.Error(t, err)
assert.Contains(t, out.String(), "My own little error")
Expand Down Expand Up @@ -137,7 +137,7 @@ func TestAutoRetryApplyAllDependentModuleRetries(t *testing.T) {
out := new(bytes.Buffer)
rootPath := copyEnvironment(t, testFixtureAutoRetryApplyAllRetries)
modulePath := util.JoinPath(rootPath, testFixtureAutoRetryApplyAllRetries)
err := runTerragruntCommand(t, "terragrunt apply-all -auto-approve --terragrunt-non-interactive --terragrunt-working-dir "+modulePath, out, os.Stderr)
err := runTerragruntCommand(t, "terragrunt apply-all -auto-approve --terragrunt-non-interactive --terragrunt-forward-tf-stdout --terragrunt-working-dir "+modulePath, out, os.Stderr)

require.NoError(t, err)
s := out.String()
Expand All @@ -154,7 +154,7 @@ func TestAutoRetryConfigurableRetries(t *testing.T) {
stderr := new(bytes.Buffer)
rootPath := copyEnvironment(t, testFixtureAutoRetryConfigurableRetries)
modulePath := util.JoinPath(rootPath, testFixtureAutoRetryConfigurableRetries)
err := runTerragruntCommand(t, "terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir "+modulePath, stdout, stderr)
err := runTerragruntCommand(t, "terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-forward-tf-stdout --terragrunt-working-dir "+modulePath, stdout, stderr)
sleeps := regexp.MustCompile("Sleeping 0s before retrying.").FindAllStringIndex(stderr.String(), -1)

require.NoError(t, err)
Expand Down
6 changes: 3 additions & 3 deletions test/integration_destroy_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ func TestTerragruntDestroyOrder(t *testing.T) {

runTerragrunt(t, "terragrunt run-all apply --terragrunt-non-interactive --terragrunt-working-dir "+rootPath)

stdout, _, err := runTerragruntCommandWithOutput(t, "terragrunt run-all destroy --terragrunt-non-interactive --terragrunt-working-dir "+rootPath)
stdout, _, err := runTerragruntCommandWithOutput(t, "terragrunt run-all destroy --terragrunt-non-interactive --terragrunt-forward-tf-stdout --terragrunt-working-dir "+rootPath)
require.NoError(t, err)
assert.Regexp(t, regexp.MustCompile(`(?smi)(?:(Module E|Module D|Module B).*){3}(?:(Module A|Module C).*){2}`), stdout)
}
Expand All @@ -54,7 +54,7 @@ func TestTerragruntApplyDestroyOrder(t *testing.T) {

runTerragrunt(t, "terragrunt run-all apply --terragrunt-non-interactive --terragrunt-working-dir "+rootPath)

stdout, _, err := runTerragruntCommandWithOutput(t, "terragrunt run-all apply -destroy --terragrunt-non-interactive --terragrunt-working-dir "+rootPath)
stdout, _, err := runTerragruntCommandWithOutput(t, "terragrunt run-all apply -destroy --terragrunt-non-interactive --terragrunt-forward-tf-stdout --terragrunt-working-dir "+rootPath)
require.NoError(t, err)
assert.Regexp(t, regexp.MustCompile(`(?smi)(?:(Module E|Module D|Module B).*){3}(?:(Module A|Module C).*){2}`), stdout)
}
Expand Down Expand Up @@ -211,7 +211,7 @@ func TestPreventDestroyDependenciesIncludedConfig(t *testing.T) {
showStderr bytes.Buffer
)

err = runTerragruntCommand(t, "terragrunt show --terragrunt-non-interactive --terragrunt-working-dir "+modulePath, &showStdout, &showStderr)
err = runTerragruntCommand(t, "terragrunt show --terragrunt-non-interactive --terragrunt-forward-tf-stdout --terragrunt-working-dir "+modulePath, &showStdout, &showStderr)
logBufferContentsLineByLine(t, showStdout, "show stdout for "+modulePath)
logBufferContentsLineByLine(t, showStderr, "show stderr for "+modulePath)

Expand Down
4 changes: 2 additions & 2 deletions test/integration_download_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -415,7 +415,7 @@ func TestTerragruntExternalDependencies(t *testing.T) {
rootPath := copyEnvironment(t, testFixtureExternalDependence)
modulePath := util.JoinPath(rootPath, testFixtureExternalDependence, "module-b")

err := runTerragruntCommand(t, "terragrunt apply-all --terragrunt-non-interactive --terragrunt-include-external-dependencies --terragrunt-working-dir "+modulePath, &applyAllStdout, &applyAllStderr)
err := runTerragruntCommand(t, "terragrunt apply-all --terragrunt-non-interactive --terragrunt-include-external-dependencies --terragrunt-forward-tf-stdout --terragrunt-working-dir "+modulePath, &applyAllStdout, &applyAllStderr)
logBufferContentsLineByLine(t, applyAllStdout, "apply-all stdout")
logBufferContentsLineByLine(t, applyAllStderr, "apply-all stderr")
applyAllStdoutString := applyAllStdout.String()
Expand Down Expand Up @@ -518,7 +518,7 @@ func TestPreventDestroyDependencies(t *testing.T) {
showStderr bytes.Buffer
)

err = runTerragruntCommand(t, "terragrunt show --terragrunt-non-interactive --terragrunt-working-dir "+modulePath, &showStdout, &showStderr)
err = runTerragruntCommand(t, "terragrunt show --terragrunt-non-interactive --terragrunt-forward-tf-stdout --terragrunt-working-dir "+modulePath, &showStdout, &showStderr)
logBufferContentsLineByLine(t, showStdout, "show stdout for "+modulePath)
logBufferContentsLineByLine(t, showStderr, "show stderr for "+modulePath)

Expand Down
12 changes: 6 additions & 6 deletions test/integration_engine_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ var LocalEngineBinaryPath = "terragrunt-iac-engine-opentofu_rpc_" + testEngineVe
func TestEngineLocalPlan(t *testing.T) {
rootPath := setupLocalEngine(t)

stdout, stderr, err := runTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt plan --terragrunt-non-interactive --terragrunt-working-dir %s --terragrunt-log-level debug", rootPath))
stdout, stderr, err := runTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt plan --terragrunt-non-interactive --terragrunt-forward-tf-stdout --terragrunt-working-dir %s --terragrunt-log-level debug", rootPath))
require.NoError(t, err)

assert.Contains(t, stderr, LocalEngineBinaryPath+": plugin address")
Expand All @@ -45,7 +45,7 @@ func TestEngineLocalPlan(t *testing.T) {
func TestEngineLocalApply(t *testing.T) {
rootPath := setupLocalEngine(t)

stdout, stderr, err := runTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir %s", rootPath))
stdout, stderr, err := runTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-forward-tf-stdout --terragrunt-working-dir %s", rootPath))
require.NoError(t, err)

assert.Contains(t, stderr, LocalEngineBinaryPath+": plugin address")
Expand All @@ -61,7 +61,7 @@ func TestEngineOpentofu(t *testing.T) {
tmpEnvPath := copyEnvironment(t, testFixtureOpenTofuEngine)
rootPath := util.JoinPath(tmpEnvPath, testFixtureOpenTofuEngine)

stdout, stderr, err := runTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir %s", rootPath))
stdout, stderr, err := runTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-forward-tf-stdout --terragrunt-working-dir %s", rootPath))
require.NoError(t, err)

assert.Contains(t, stderr, "starting plugin:")
Expand All @@ -77,7 +77,7 @@ func TestEngineRunAllOpentofu(t *testing.T) {
tmpEnvPath := copyEnvironment(t, testFixtureOpenTofuRunAll)
rootPath := util.JoinPath(tmpEnvPath, testFixtureOpenTofuRunAll)

stdout, stderr, err := runTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt run-all apply -no-color -auto-approve --terragrunt-non-interactive --terragrunt-working-dir %s", rootPath))
stdout, stderr, err := runTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt run-all apply -no-color -auto-approve --terragrunt-non-interactive --terragrunt-forward-tf-stdout --terragrunt-working-dir %s", rootPath))
require.NoError(t, err)

assert.Contains(t, stderr, "starting plugin:")
Expand All @@ -94,7 +94,7 @@ func TestEngineRunAllOpentofuCustomPath(t *testing.T) {

cacheDir, rootPath := setupEngineCache(t)

stdout, stderr, err := runTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt run-all apply -no-color -auto-approve --terragrunt-non-interactive --terragrunt-working-dir %s", rootPath))
stdout, stderr, err := runTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt run-all apply -no-color -auto-approve --terragrunt-non-interactive --terragrunt-forward-tf-stdout --terragrunt-working-dir %s", rootPath))
require.NoError(t, err)

assert.Contains(t, stderr, "starting plugin:")
Expand Down Expand Up @@ -123,7 +123,7 @@ func TestEngineDownloadOverHttp(t *testing.T) {
"__hardcoded_url__": fmt.Sprintf("https://github.com/gruntwork-io/terragrunt-engine-opentofu/releases/download/v0.0.4/terragrunt-iac-engine-opentofu_rpc_v0.0.4_%s_%s.zip", platform, arch),
})

stdout, stderr, err := runTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir %s", rootPath))
stdout, stderr, err := runTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-forward-tf-stdout --terragrunt-working-dir %s", rootPath))
require.NoError(t, err)

assert.Contains(t, stderr, "starting plugin:")
Expand Down
6 changes: 3 additions & 3 deletions test/integration_include_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ func TestTerragruntRunAllModulesThatIncludeRestrictsSet(t *testing.T) {
err := runTerragruntCommand(
t,
fmt.Sprintf(
"terragrunt run-all plan --terragrunt-non-interactive --terragrunt-log-level debug --terragrunt-working-dir %s --terragrunt-modules-that-include alpha.hcl",
"terragrunt run-all plan --terragrunt-non-interactive --terragrunt-log-level debug --terragrunt-forward-tf-stdout --terragrunt-working-dir %s --terragrunt-modules-that-include alpha.hcl",
modulePath,
),
&stdout,
Expand All @@ -132,7 +132,7 @@ func TestTerragruntRunAllModulesWithPrefix(t *testing.T) {
stderr := bytes.Buffer{}
err := runTerragruntCommand(
t,
"terragrunt run-all plan --terragrunt-non-interactive --terragrunt-working-dir "+modulePath,
"terragrunt run-all plan --terragrunt-non-interactive --terragrunt-forward-tf-stdout --terragrunt-working-dir "+modulePath,
&stdout,
&stderr,
)
Expand All @@ -145,7 +145,7 @@ func TestTerragruntRunAllModulesWithPrefix(t *testing.T) {
assert.Contains(t, planOutput, "beta")
assert.Contains(t, planOutput, "charlie")

stdoutLines := strings.Split(planOutput, "\n")
stdoutLines := strings.Split(stderr.String(), "\n")
for _, line := range stdoutLines {
if strings.Contains(line, "alpha") {
assert.Contains(t, line, "prefix=a")
Expand Down
Loading

0 comments on commit 64baa28

Please sign in to comment.