From e37d71e9cfebb512a61e904500df8ee957fca201 Mon Sep 17 00:00:00 2001 From: Levko Burburas <62853952+levkohimins@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:53:03 +0300 Subject: [PATCH] Added `hclvalidate` command (#3248) * chore: configstack code refactoring * fix: tests * fix: tests * fix: hclparse options * fix: parse config * fix: unit test * fix: unit test * chore: print diagnostics in a human-friendly format * fix: lint * chore: fix unit tests, improve locals diagnostic * fix: unit test * chore: add integration test * fix: fixture * chore: disable SONAR for unit tests * fix: test * chore: sonar properties * chore: code improvements * chore: update sonarcloud properties * chore: update docs * fix: grammar * chore: update docs * chore: custom errors renaming * chore: code improvements * fix: review commetns --- .sonarcloud.properties | 4 + cli/app.go | 7 + cli/commands/graph-dependencies/action.go | 2 +- cli/commands/graph/action.go | 4 +- cli/commands/hclfmt/action.go | 6 +- cli/commands/hclvalidate/action.go | 66 + cli/commands/hclvalidate/command.go | 47 + cli/commands/hclvalidate/options.go | 16 + cli/commands/output-module-groups/action.go | 2 +- cli/commands/run-all/action.go | 2 +- cli/commands/scaffold/action_test.go | 3 +- cli/commands/terraform/action.go | 2 +- cli/provider_cache.go | 4 - config/config.go | 27 +- config/config_helpers.go | 4 +- config/config_partial.go | 8 +- config/config_test.go | 2 +- config/dependency.go | 13 +- config/dependency_test.go | 7 +- config/errors.go | 4 +- config/hclparse/attributes.go | 22 +- config/hclparse/block.go | 2 +- config/hclparse/file.go | 30 +- config/hclparse/options.go | 59 +- config/hclparse/parser.go | 45 +- config/include.go | 3 +- config/locals.go | 110 +- config/parsing_context.go | 8 + configstack/errors.go | 61 +- configstack/graph.go | 51 - configstack/graph_test.go | 87 - configstack/graphviz.go | 62 - configstack/graphviz_test.go | 115 -- configstack/log.go | 44 + configstack/log_test.go | 50 + configstack/module.go | 877 +++----- configstack/module_test.go | 1761 +++++++++-------- configstack/options.go | 22 + configstack/running_module.go | 338 ++-- configstack/running_module_test.go | 1219 +----------- configstack/stack.go | 623 ++++-- configstack/stack_test.go | 1034 +++++++++- configstack/test_helpers.go | 12 +- docs/_docs/04_reference/cli-options.md | 39 + go.mod | 10 +- go.sum | 16 +- internal/view/diagnostic/diagnostic.go | 58 + internal/view/diagnostic/expression_value.go | 164 ++ internal/view/diagnostic/extra.go | 73 + internal/view/diagnostic/function.go | 120 ++ internal/view/diagnostic/range.go | 40 + internal/view/diagnostic/servity.go | 41 + internal/view/diagnostic/snippet.go | 95 + internal/view/human_render.go | 271 +++ internal/view/json_render.go | 36 + internal/view/writer.go | 64 + options/options.go | 4 + .../first/b/terragrunt.hcl | 3 + test/fixture-hclvalidate/second/a/main.tf | 8 + .../second/a/terragrunt.hcl | 3 + test/fixture-hclvalidate/second/c/main.tf | 8 + .../second/c/terragrunt.hcl | 13 + test/fixture-hclvalidate/second/d/main.tf | 8 + .../second/d/terragrunt.hcl | 11 + .../b/terragrunt_rendered.json | 1 - .../d/terragrunt_rendered.json | 1 - test/integration_catalog_test.go | 2 +- test/integration_serial_test.go | 4 - test/integration_test.go | 117 +- util/logger.go | 5 +- 70 files changed, 4770 insertions(+), 3310 deletions(-) create mode 100644 .sonarcloud.properties create mode 100644 cli/commands/hclvalidate/action.go create mode 100644 cli/commands/hclvalidate/command.go create mode 100644 cli/commands/hclvalidate/options.go delete mode 100644 configstack/graph.go delete mode 100644 configstack/graph_test.go delete mode 100644 configstack/graphviz.go delete mode 100644 configstack/graphviz_test.go create mode 100644 configstack/log.go create mode 100644 configstack/log_test.go create mode 100644 configstack/options.go create mode 100644 internal/view/diagnostic/diagnostic.go create mode 100644 internal/view/diagnostic/expression_value.go create mode 100644 internal/view/diagnostic/extra.go create mode 100644 internal/view/diagnostic/function.go create mode 100644 internal/view/diagnostic/range.go create mode 100644 internal/view/diagnostic/servity.go create mode 100644 internal/view/diagnostic/snippet.go create mode 100644 internal/view/human_render.go create mode 100644 internal/view/json_render.go create mode 100644 internal/view/writer.go create mode 100644 test/fixture-hclvalidate/first/b/terragrunt.hcl create mode 100644 test/fixture-hclvalidate/second/a/main.tf create mode 100644 test/fixture-hclvalidate/second/a/terragrunt.hcl create mode 100644 test/fixture-hclvalidate/second/c/main.tf create mode 100644 test/fixture-hclvalidate/second/c/terragrunt.hcl create mode 100644 test/fixture-hclvalidate/second/d/main.tf create mode 100644 test/fixture-hclvalidate/second/d/terragrunt.hcl delete mode 100644 test/fixutre-excludes-file/b/terragrunt_rendered.json delete mode 100644 test/fixutre-excludes-file/d/terragrunt_rendered.json diff --git a/.sonarcloud.properties b/.sonarcloud.properties new file mode 100644 index 000000000..de631a32d --- /dev/null +++ b/.sonarcloud.properties @@ -0,0 +1,4 @@ +# Source File Exclusions: Patterns used to exclude some source files from analysis. +sonar.exclusions=**/*_test.go +# Test File Inclusions: Patterns used to include some test files and only these ones in analysis. +sonar.test.inclusions=**/*_test.go diff --git a/cli/app.go b/cli/app.go index 3e1429881..c868f177f 100644 --- a/cli/app.go +++ b/cli/app.go @@ -18,6 +18,7 @@ import ( "golang.org/x/text/language" "github.com/gruntwork-io/terragrunt/cli/commands/graph" + "github.com/gruntwork-io/terragrunt/cli/commands/hclvalidate" "github.com/gruntwork-io/terragrunt/cli/commands/scaffold" @@ -97,6 +98,11 @@ func (app *App) RunContext(ctx context.Context, args []string) error { log.Infof("%s signal received. Gracefully shutting down... (it can take up to %v)", cases.Title(language.English).String(signal.String()), shell.SignalForwardingDelay) cancel() + shell.RegisterSignalHandler(func(signal os.Signal) { + log.Infof("Second %s signal received, force shutting down...", cases.Title(language.English).String(signal.String())) + os.Exit(1) + }) + time.Sleep(forceExitInterval) log.Infof("Failed to gracefully shutdown within %v, force shutting down...", forceExitInterval) os.Exit(1) @@ -139,6 +145,7 @@ func terragruntCommands(opts *options.TerragruntOptions) cli.Commands { catalog.NewCommand(opts), // catalog scaffold.NewCommand(opts), // scaffold graph.NewCommand(opts), // graph + hclvalidate.NewCommand(opts), // hclvalidate } sort.Sort(cmds) diff --git a/cli/commands/graph-dependencies/action.go b/cli/commands/graph-dependencies/action.go index 5e0892101..dff7000fd 100644 --- a/cli/commands/graph-dependencies/action.go +++ b/cli/commands/graph-dependencies/action.go @@ -9,7 +9,7 @@ import ( // Run graph dependencies prints the dependency graph to stdout func Run(ctx context.Context, opts *options.TerragruntOptions) error { - stack, err := configstack.FindStackInSubfolders(ctx, opts, nil) + stack, err := configstack.FindStackInSubfolders(ctx, opts) if err != nil { return err } diff --git a/cli/commands/graph/action.go b/cli/commands/graph/action.go index 8d2bcb94e..b16cbdfd4 100644 --- a/cli/commands/graph/action.go +++ b/cli/commands/graph/action.go @@ -41,11 +41,11 @@ func graph(ctx context.Context, opts *options.TerragruntOptions, cfg *config.Ter rootOptions := opts.Clone(rootDir) rootOptions.WorkingDir = rootDir - stack, err := configstack.FindStackInSubfolders(ctx, rootOptions, nil) + stack, err := configstack.FindStackInSubfolders(ctx, rootOptions) if err != nil { return err } - dependentModules := configstack.ListStackDependentModules(stack) + dependentModules := stack.ListStackDependentModules() workDir := opts.WorkingDir modulesToInclude := dependentModules[workDir] diff --git a/cli/commands/hclfmt/action.go b/cli/commands/hclfmt/action.go index 9b905290c..32a919275 100644 --- a/cli/commands/hclfmt/action.go +++ b/cli/commands/hclfmt/action.go @@ -92,7 +92,7 @@ func formatTgHCL(opts *options.TerragruntOptions, tgHclFile string) error { } contents := []byte(contentsStr) - err = checkErrors(opts.Logger, contents, tgHclFile) + err = checkErrors(opts.Logger, opts.DisableLogColors, contents, tgHclFile) if err != nil { opts.Logger.Errorf("Error parsing %s", tgHclFile) return err @@ -128,10 +128,10 @@ func formatTgHCL(opts *options.TerragruntOptions, tgHclFile string) error { } // checkErrors takes in the contents of a hcl file and looks for syntax errors. -func checkErrors(logger *logrus.Entry, contents []byte, tgHclFile string) error { +func checkErrors(logger *logrus.Entry, disableColor bool, contents []byte, tgHclFile string) error { parser := hclparse.NewParser() _, diags := parser.ParseHCL(contents, tgHclFile) - diagWriter := util.GetDiagnosticsWriter(logger, parser) + diagWriter := util.GetDiagnosticsWriter(logger, parser, disableColor) err := diagWriter.WriteDiagnostics(diags) if err != nil { return errors.WithStackTrace(err) diff --git a/cli/commands/hclvalidate/action.go b/cli/commands/hclvalidate/action.go new file mode 100644 index 000000000..58813caff --- /dev/null +++ b/cli/commands/hclvalidate/action.go @@ -0,0 +1,66 @@ +package hclvalidate + +import ( + "context" + + "github.com/gruntwork-io/terragrunt/config" + "github.com/gruntwork-io/terragrunt/config/hclparse" + "github.com/gruntwork-io/terragrunt/configstack" + "github.com/gruntwork-io/terragrunt/internal/view" + "github.com/gruntwork-io/terragrunt/internal/view/diagnostic" + "github.com/gruntwork-io/terragrunt/options" + "github.com/hashicorp/hcl/v2" +) + +func Run(ctx context.Context, opts *Options) (er error) { + var diags diagnostic.Diagnostics + + parseOptions := []hclparse.Option{ + hclparse.WithDiagnosticsHandler(func(file *hcl.File, hclDiags hcl.Diagnostics) (hcl.Diagnostics, error) { + for _, hclDiag := range hclDiags { + if !diags.Contains(hclDiag) { + newDiag := diagnostic.NewDiagnostic(file, hclDiag) + diags = append(diags, newDiag) + } + } + return nil, nil + }), + } + + opts.SkipOutput = true + opts.NonInteractive = true + opts.RunTerragrunt = func(ctx context.Context, opts *options.TerragruntOptions) error { + _, err := config.ReadTerragruntConfig(ctx, opts, parseOptions) + return err + } + + stack, err := configstack.FindStackInSubfolders(ctx, opts.TerragruntOptions, configstack.WithParseOptions(parseOptions)) + if err != nil { + return err + } + + stackErr := stack.Run(ctx, opts.TerragruntOptions) + + if len(diags) > 0 { + if err := writeDiagnostics(opts, diags); err != nil { + return err + } + } + + return stackErr +} + +func writeDiagnostics(opts *Options, diags diagnostic.Diagnostics) error { + render := view.NewHumanRender(opts.DisableLogColors) + if opts.JSONOutput { + render = view.NewJSONRender() + } + + writer := view.NewWriter(opts.Writer, render) + + if opts.InvalidConfigPath { + return writer.InvalidConfigPath(diags) + } + + return writer.Diagnostics(diags) +} diff --git a/cli/commands/hclvalidate/command.go b/cli/commands/hclvalidate/command.go new file mode 100644 index 000000000..00410c086 --- /dev/null +++ b/cli/commands/hclvalidate/command.go @@ -0,0 +1,47 @@ +// `hclvalidate` command recursively looks for hcl files in the directory tree starting at workingDir, and validates them +// based on the language style guides provided by Hashicorp. This is done using the official hcl2 library. + +package hclvalidate + +import ( + "github.com/gruntwork-io/terragrunt/options" + "github.com/gruntwork-io/terragrunt/pkg/cli" +) + +const ( + CommandName = "hclvalidate" + + InvalidFlagName = "terragrunt-hclvalidate-invalid" + InvalidEnvVarName = "TERRAGRUNT_HCLVALIDATE_INVALID" + + JSONOutputFlagName = "terragrunt-hclvalidate-json" + JSONOutputEnvVarName = "TERRAGRUNT_HCLVALIDATE_JSON" +) + +func NewFlags(opts *Options) cli.Flags { + return cli.Flags{ + &cli.BoolFlag{ + Name: InvalidFlagName, + EnvVar: InvalidEnvVarName, + Usage: "Show a list of files with invalid configuration.", + Destination: &opts.InvalidConfigPath, + }, + &cli.BoolFlag{ + Name: JSONOutputFlagName, + EnvVar: JSONOutputEnvVarName, + Destination: &opts.JSONOutput, + Usage: "Output the result in JSON format.", + }, + } +} + +func NewCommand(generalOpts *options.TerragruntOptions) *cli.Command { + opts := NewOptions(generalOpts) + + return &cli.Command{ + Name: CommandName, + Usage: "Find all hcl files from the config stack and validate them.", + Flags: NewFlags(opts).Sort(), + Action: func(ctx *cli.Context) error { return Run(ctx, opts) }, + } +} diff --git a/cli/commands/hclvalidate/options.go b/cli/commands/hclvalidate/options.go new file mode 100644 index 000000000..dee46ff2a --- /dev/null +++ b/cli/commands/hclvalidate/options.go @@ -0,0 +1,16 @@ +package hclvalidate + +import "github.com/gruntwork-io/terragrunt/options" + +type Options struct { + *options.TerragruntOptions + + InvalidConfigPath bool + JSONOutput bool +} + +func NewOptions(general *options.TerragruntOptions) *Options { + return &Options{ + TerragruntOptions: general, + } +} diff --git a/cli/commands/output-module-groups/action.go b/cli/commands/output-module-groups/action.go index 19307202b..06c1e6df3 100644 --- a/cli/commands/output-module-groups/action.go +++ b/cli/commands/output-module-groups/action.go @@ -9,7 +9,7 @@ import ( ) func Run(ctx context.Context, opts *options.TerragruntOptions) error { - stack, err := configstack.FindStackInSubfolders(ctx, opts, nil) + stack, err := configstack.FindStackInSubfolders(ctx, opts) if err != nil { return err } diff --git a/cli/commands/run-all/action.go b/cli/commands/run-all/action.go index aa43c958f..573d964fd 100644 --- a/cli/commands/run-all/action.go +++ b/cli/commands/run-all/action.go @@ -42,7 +42,7 @@ func Run(ctx context.Context, opts *options.TerragruntOptions) error { } } - stack, err := configstack.FindStackInSubfolders(ctx, opts, nil) + stack, err := configstack.FindStackInSubfolders(ctx, opts) if err != nil { return err } diff --git a/cli/commands/scaffold/action_test.go b/cli/commands/scaffold/action_test.go index a98b3f9e4..8d7304cc3 100644 --- a/cli/commands/scaffold/action_test.go +++ b/cli/commands/scaffold/action_test.go @@ -1,6 +1,7 @@ package scaffold import ( + "context" "os" "path/filepath" "testing" @@ -81,7 +82,7 @@ func TestDefaultTemplateVariables(t *testing.T) { opts, err := options.NewTerragruntOptionsForTest(filepath.Join(outputDir, "terragrunt.hcl")) require.NoError(t, err) - cfg, err := config.ReadTerragruntConfig(opts) + cfg, err := config.ReadTerragruntConfig(context.Background(), opts, config.DefaultParserOptions(opts)) require.NoError(t, err) require.NotEmpty(t, cfg.Inputs) require.Equal(t, 1, len(cfg.Inputs)) diff --git a/cli/commands/terraform/action.go b/cli/commands/terraform/action.go index e5bc6a9e0..8fb52fa4c 100644 --- a/cli/commands/terraform/action.go +++ b/cli/commands/terraform/action.go @@ -95,7 +95,7 @@ func runTerraform(ctx context.Context, terragruntOptions *options.TerragruntOpti return err } - terragruntConfig, err := config.ReadTerragruntConfig(terragruntOptions) + terragruntConfig, err := config.ReadTerragruntConfig(ctx, terragruntOptions, config.DefaultParserOptions(terragruntOptions)) if err != nil { return target.runErrorCallback(terragruntOptions, terragruntConfig, err) } diff --git a/cli/provider_cache.go b/cli/provider_cache.go index 818c887d7..c9598a600 100644 --- a/cli/provider_cache.go +++ b/cli/provider_cache.go @@ -193,10 +193,6 @@ func (cache *ProviderCache) TerraformCommandHook(ctx context.Context, opts *opti cloneOpts.WorkingDir = opts.WorkingDir maps.Copy(cloneOpts.Env, env) - if util.FirstArg(args) == terraform.CommandNameProviders && util.SecondArg(args) == terraform.CommandNameLock { - return &shell.CmdOutput{}, nil - } - if skipRunTargetCommand { return &shell.CmdOutput{}, nil } diff --git a/config/config.go b/config/config.go index 3cf21af7d..202924544 100644 --- a/config/config.go +++ b/config/config.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/gruntwork-io/terragrunt/internal/cache" + "github.com/gruntwork-io/terragrunt/shell" "github.com/gruntwork-io/terragrunt/telemetry" "github.com/mitchellh/mapstructure" @@ -72,7 +73,7 @@ var ( DefaultParserOptions = func(opts *options.TerragruntOptions) []hclparse.Option { return []hclparse.Option{ - hclparse.WithLogger(opts.Logger), + hclparse.WithLogger(opts.Logger, opts.DisableLogColors), hclparse.WithFileUpdate(updateBareIncludeBlock), } } @@ -680,23 +681,23 @@ func isTerragruntModuleDir(path string, terragruntOptions *options.TerragruntOpt } // Read the Terragrunt config file from its default location -func ReadTerragruntConfig(terragruntOptions *options.TerragruntOptions) (*TerragruntConfig, error) { +func ReadTerragruntConfig(ctx context.Context, terragruntOptions *options.TerragruntOptions, parserOptions []hclparse.Option) (*TerragruntConfig, error) { terragruntOptions.Logger.Debugf("Reading Terragrunt config file at %s", terragruntOptions.TerragruntConfigPath) - ctx := NewParsingContext(context.Background(), terragruntOptions) - return ParseConfigFile(terragruntOptions, ctx, terragruntOptions.TerragruntConfigPath, nil) + ctx = shell.ContextWithTerraformCommandHook(ctx, nil) + parcingCtx := NewParsingContext(ctx, terragruntOptions).WithParseOption(parserOptions) + return ParseConfigFile(parcingCtx, terragruntOptions.TerragruntConfigPath, nil) } var hclCache = cache.NewCache[*hclparse.File]() // Parse the Terragrunt config file at the given path. If the include parameter is not nil, then treat this as a config // included in some other config file when resolving relative paths. -func ParseConfigFile(opts *options.TerragruntOptions, ctx *ParsingContext, configPath string, includeFromChild *IncludeConfig) (*TerragruntConfig, error) { - +func ParseConfigFile(ctx *ParsingContext, configPath string, includeFromChild *IncludeConfig) (*TerragruntConfig, error) { var config *TerragruntConfig - err := telemetry.Telemetry(ctx, opts, "parse_config_file", map[string]interface{}{ + err := telemetry.Telemetry(ctx, ctx.TerragruntOptions, "parse_config_file", map[string]interface{}{ "config_path": configPath, - "working_dir": opts.WorkingDir, + "working_dir": ctx.TerragruntOptions.WorkingDir, }, func(childCtx context.Context) error { childKey := "nil" if includeFromChild != nil { @@ -716,7 +717,7 @@ func ParseConfigFile(opts *options.TerragruntOptions, ctx *ParsingContext, confi return err } var file *hclparse.File - var cacheKey = fmt.Sprintf("parse-config-%v-%v-%v-%v-%v-%v", configPath, childKey, decodeListKey, opts.WorkingDir, dir, fileInfo.ModTime().UnixMicro()) + var cacheKey = fmt.Sprintf("parse-config-%v-%v-%v-%v-%v-%v", configPath, childKey, decodeListKey, ctx.TerragruntOptions.WorkingDir, dir, fileInfo.ModTime().UnixMicro()) if cacheConfig, found := hclCache.Get(cacheKey); found { file = cacheConfig } else { @@ -888,7 +889,9 @@ func decodeAsTerragruntConfigFile(ctx *ParsingContext, file *hclparse.File, eval return nil, err } ctx.TerragruntOptions.Logger.Warnf("Failed to decode inputs %v", diagErr) + } + if terragruntConfig.Inputs != nil { inputs, err := updateUnknownCtyValValues(terragruntConfig.Inputs) if err != nil { return nil, err @@ -1143,6 +1146,12 @@ func convertToTerragruntConfig(ctx *ParsingContext, configPath string, terragrun } if ctx.Locals != nil && *ctx.Locals != cty.NilVal { + locals, err := updateUnknownCtyValValues(ctx.Locals) + if err != nil { + return nil, err + } + ctx.Locals = locals + localsParsed, err := parseCtyValueToMap(*ctx.Locals) if err != nil { return nil, err diff --git a/config/config_helpers.go b/config/config_helpers.go index 8527586ea..4f79cf1ec 100644 --- a/config/config_helpers.go +++ b/config/config_helpers.go @@ -519,7 +519,7 @@ func getWorkingDir(ctx *ParsingContext) (string, error) { FuncNameGetWorkingDir: wrapVoidToEmptyStringAsFuncImpl(), } - terragruntConfig, err := ParseConfigFile(ctx.TerragruntOptions, ctx, ctx.TerragruntOptions.TerragruntConfigPath, nil) + terragruntConfig, err := ParseConfigFile(ctx, ctx.TerragruntOptions.TerragruntConfigPath, nil) if err != nil { return "", err } @@ -594,7 +594,7 @@ func readTerragruntConfig(ctx *ParsingContext, configPath string, defaultVal *ct // We update the ctx of terragruntOptions to the config being read in. ctx = ctx.WithTerragruntOptions(ctx.TerragruntOptions.Clone(targetConfig)) - config, err := ParseConfigFile(ctx.TerragruntOptions, ctx, targetConfig, nil) + config, err := ParseConfigFile(ctx, targetConfig, nil) if err != nil { return cty.NilVal, err } diff --git a/config/config_partial.go b/config/config_partial.go index f300547fc..1aada2cf2 100644 --- a/config/config_partial.go +++ b/config/config_partial.go @@ -314,15 +314,15 @@ func PartialParseConfig(ctx *ParsingContext, file *hclparse.File, includeFromChi return nil, err } ctx.TerragruntOptions.Logger.Warnf("Failed to decode inputs %v", diagErr) + } - inputs, err := updateUnknownCtyValValues(decoded.Inputs) + if decoded.Inputs != nil { + val, err := updateUnknownCtyValValues(decoded.Inputs) if err != nil { return nil, err } - decoded.Inputs = inputs - } + decoded.Inputs = val - if decoded.Inputs != nil { inputs, err := parseCtyValueToMap(*decoded.Inputs) if err != nil { return nil, err diff --git a/config/config_test.go b/config/config_test.go index ef8c8aa81..e46f883ad 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1356,7 +1356,7 @@ func BenchmarkReadTerragruntConfig(b *testing.B) { b.ResetTimer() b.StartTimer() - actual, err := ReadTerragruntConfig(terragruntOptions) + actual, err := ReadTerragruntConfig(context.Background(), terragruntOptions, DefaultParserOptions(terragruntOptions)) b.StopTimer() require.NoError(b, err) require.NotNil(b, actual) diff --git a/config/dependency.go b/config/dependency.go index 93d3474d8..546794a99 100644 --- a/config/dependency.go +++ b/config/dependency.go @@ -114,8 +114,8 @@ func (dependencyConfig Dependency) getMockOutputsMergeStrategy() MergeStrategyTy } // Given a dependency config, we should only attempt to get the outputs if SkipOutputs is nil or false -func (dependencyConfig Dependency) shouldGetOutputs() bool { - return dependencyConfig.isEnabled() && (dependencyConfig.SkipOutputs == nil || !*dependencyConfig.SkipOutputs) +func (dependencyConfig Dependency) shouldGetOutputs(ctx *ParsingContext) bool { + return !ctx.TerragruntOptions.SkipOutput && dependencyConfig.isEnabled() && (dependencyConfig.SkipOutputs == nil || !*dependencyConfig.SkipOutputs) } // isEnabled returns true if the dependency is enabled @@ -145,7 +145,7 @@ func (dependencyConfig *Dependency) setRenderedOutputs(ctx *ParsingContext) erro return nil } - if dependencyConfig.shouldGetOutputs() || dependencyConfig.shouldReturnMockOutputs(ctx) { + if dependencyConfig.shouldGetOutputs(ctx) || dependencyConfig.shouldReturnMockOutputs(ctx) { outputVal, err := getTerragruntOutputIfAppliedElseConfiguredDefault(ctx, *dependencyConfig) if err != nil { return err @@ -278,7 +278,7 @@ func checkForDependencyBlockCyclesUsingDFS( } if util.ListContainsElement(*currentTraversalPaths, dependencyPath) { - return errors.WithStackTrace(DependencyCycle(append(*currentTraversalPaths, dependencyPath))) + return errors.WithStackTrace(DependencyCycleError(append(*currentTraversalPaths, dependencyPath))) } *currentTraversalPaths = append(*currentTraversalPaths, dependencyPath) @@ -343,6 +343,7 @@ func dependencyBlocksToCtyValue(ctx *ParsingContext, dependencyConfigs []Depende if err := dependencyConfig.setRenderedOutputs(ctx); err != nil { return err } + if dependencyConfig.RenderedOutputs != nil { lock.Lock() paths = append(paths, dependencyConfig.ConfigPath) @@ -394,7 +395,8 @@ func getTerragruntOutputIfAppliedElseConfiguredDefault(ctx *ParsingContext, depe ctx.TerragruntOptions.Logger.Debugf("Skipping outputs reading for disabled dependency %s", dependencyConfig.Name) return dependencyConfig.MockOutputs, nil } - if dependencyConfig.shouldGetOutputs() { + + if dependencyConfig.shouldGetOutputs(ctx) { outputVal, isEmpty, err := getTerragruntOutput(ctx, dependencyConfig) if err != nil { return nil, err @@ -970,6 +972,7 @@ func runTerraformInitForDependencyOutput(ctx *ParsingContext, workingDir string, initTGOptions := cloneTerragruntOptionsForDependency(ctx, targetConfigPath) initTGOptions.WorkingDir = workingDir initTGOptions.ErrWriter = &stderr + err := shell.RunTerraformCommand(ctx, initTGOptions, terraform.CommandNameInit, "-get=false") if err != nil { ctx.TerragruntOptions.Logger.Debugf("Ignoring expected error from dependency init call") diff --git a/config/dependency_test.go b/config/dependency_test.go index 5129a142f..a90afa0d3 100644 --- a/config/dependency_test.go +++ b/config/dependency_test.go @@ -116,11 +116,12 @@ func TestParseDependencyBlockMultiple(t *testing.T) { filename := "../test/fixture-regressions/multiple-dependency-load-sync/main/terragrunt.hcl" ctx := NewParsingContext(context.Background(), mockOptionsForTestWithConfigPath(t, filename)) - ctx.TerragruntOptions.FetchDependencyOutputFromState = true - ctx.TerragruntOptions.Env = env.Parse(os.Environ()) opts, err := options.NewTerragruntOptionsForTest(filename) require.NoError(t, err) - tfConfig, err := ParseConfigFile(opts, ctx, filename, nil) + ctx.TerragruntOptions = opts + ctx.TerragruntOptions.FetchDependencyOutputFromState = true + ctx.TerragruntOptions.Env = env.Parse(os.Environ()) + tfConfig, err := ParseConfigFile(ctx, filename, nil) require.NoError(t, err) require.Len(t, tfConfig.TerragruntDependencies, 2) assert.Equal(t, tfConfig.TerragruntDependencies[0].Name, "dependency_1") diff --git a/config/errors.go b/config/errors.go index 148f77cfa..d42d50a6a 100644 --- a/config/errors.go +++ b/config/errors.go @@ -238,8 +238,8 @@ func (err TerragruntOutputTargetNoOutputs) Error() string { ) } -type DependencyCycle []string +type DependencyCycleError []string -func (err DependencyCycle) Error() string { +func (err DependencyCycleError) Error() string { return fmt.Sprintf("Found a dependency cycle between modules: %s", strings.Join([]string(err), " -> ")) } diff --git a/config/hclparse/attributes.go b/config/hclparse/attributes.go index 271ad9c6f..ae1e2aeb3 100644 --- a/config/hclparse/attributes.go +++ b/config/hclparse/attributes.go @@ -32,6 +32,24 @@ func (attrs Attributes) ValidateIdentifier() error { return nil } +func (attrs Attributes) Range() hcl.Range { + var rng hcl.Range + + for _, attr := range attrs { + rng.Filename = attr.Range.Filename + + if rng.Start.Line > attr.Range.Start.Line || rng.Start.Column > attr.Range.Start.Column { + rng.Start = attr.Range.Start + } + + if rng.End.Line < attr.Range.End.Line || rng.End.Column < attr.Range.End.Column { + rng.End = attr.Range.End + } + } + + return rng +} + type Attribute struct { *File *hcl.Attribute @@ -46,7 +64,7 @@ func (attr *Attribute) ValidateIdentifier() error { Subject: &attr.NameRange, }} - if err := attr.diagnosticsError(diags); err != nil { + if err := attr.HandleDiagnostics(diags); err != nil { return errors.WithStackTrace(err) } } @@ -57,7 +75,7 @@ func (attr *Attribute) ValidateIdentifier() error { func (attr *Attribute) Value(evalCtx *hcl.EvalContext) (cty.Value, error) { evaluatedVal, diags := attr.Expr.Value(evalCtx) - if err := attr.diagnosticsError(diags); err != nil { + if err := attr.HandleDiagnostics(diags); err != nil { return evaluatedVal, errors.WithStackTrace(err) } diff --git a/config/hclparse/block.go b/config/hclparse/block.go index 88b100077..7a6e59a1d 100644 --- a/config/hclparse/block.go +++ b/config/hclparse/block.go @@ -23,7 +23,7 @@ type Block struct { func (block *Block) JustAttributes() (Attributes, error) { hclAttrs, diags := block.Body.JustAttributes() - if err := block.diagnosticsError(diags); err != nil { + if err := block.HandleDiagnostics(diags); err != nil { return nil, errors.WithStackTrace(err) } diff --git a/config/hclparse/file.go b/config/hclparse/file.go index 5101f5db3..060de6c21 100644 --- a/config/hclparse/file.go +++ b/config/hclparse/file.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/gruntwork-io/go-commons/errors" - "github.com/gruntwork-io/terragrunt/util" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/gohcl" "github.com/hashicorp/hcl/v2/hclparse" @@ -64,7 +63,7 @@ func (file *File) Decode(out interface{}, evalContext *hcl.EvalContext) (err err } diags := gohcl.DecodeBody(file.Body, evalContext, out) - if err := file.diagnosticsError(diags); err != nil { + if err := file.HandleDiagnostics(diags); err != nil { return errors.WithStackTrace(err) } @@ -80,7 +79,7 @@ func (file *File) Blocks(name string, isMultipleAllowed bool) ([]*Block, error) } // We use PartialContent here, because we are only interested in parsing out the catalog block. parsed, _, diags := file.Body.PartialContent(catalogSchema) - if err := file.diagnosticsError(diags); err != nil { + if err := file.HandleDiagnostics(diags); err != nil { return nil, errors.WithStackTrace(err) } @@ -110,7 +109,7 @@ func (file *File) Blocks(name string, isMultipleAllowed bool) ([]*Block, error) func (file *File) JustAttributes() (Attributes, error) { hclAttrs, diags := file.Body.JustAttributes() - if err := file.diagnosticsError(diags); err != nil { + if err := file.HandleDiagnostics(diags); err != nil { return nil, errors.WithStackTrace(err) } @@ -123,25 +122,6 @@ func (file *File) JustAttributes() (Attributes, error) { return attrs, nil } -func (file *File) diagnosticsError(diags hcl.Diagnostics) error { - if diags == nil || !diags.HasErrors() { - return nil - } - - if fn := file.Parser.diagnosticsErrorFunc; fn != nil { - var err error - if diags, err = fn(file, diags); err != nil || diags == nil { - return err - } - } - - if logger := file.Parser.logger; logger != nil { - diagsWriter := util.GetDiagnosticsWriter(logger, file.Parser.Parser) - - if err := diagsWriter.WriteDiagnostics(diags); err != nil { - return errors.WithStackTrace(err) - } - } - - return diags +func (file *File) HandleDiagnostics(diags hcl.Diagnostics) error { + return file.Parser.handleDiagnostics(file, diags) } diff --git a/config/hclparse/options.go b/config/hclparse/options.go index 4032e2df2..e8a4843af 100644 --- a/config/hclparse/options.go +++ b/config/hclparse/options.go @@ -1,22 +1,35 @@ package hclparse import ( + "github.com/gruntwork-io/go-commons/errors" + "github.com/gruntwork-io/terragrunt/util" "github.com/hashicorp/hcl/v2" "github.com/sirupsen/logrus" ) -type Option func(Parser) Parser +type Option func(*Parser) *Parser -func WithLogger(logger *logrus.Entry) Option { - return func(parser Parser) Parser { - parser.logger = logger +func WithLogger(logger *logrus.Entry, disableColor bool) Option { + return func(parser *Parser) *Parser { + diagsWriter := util.GetDiagnosticsWriter(logger, parser.Parser, disableColor) + + parser.loggerFunc = func(diags hcl.Diagnostics) error { + if !diags.HasErrors() { + return nil + } + + if err := diagsWriter.WriteDiagnostics(diags); err != nil { + return errors.WithStackTrace(err) + } + return nil + } return parser } } // WithFileUpdate sets the `fileUpdateHandlerFunc` func which is run before each file decoding. func WithFileUpdate(fn func(*File) error) Option { - return func(parser Parser) Parser { + return func(parser *Parser) *Parser { parser.fileUpdateHandlerFunc = fn return parser } @@ -25,10 +38,14 @@ func WithFileUpdate(fn func(*File) error) Option { // WithHaltOnErrorOnlyForBlocks configures a diagnostic error handler that runs when diagnostic errors occur. // If errors occur in the given `blockNames` blocks, parser returns the error to its caller, otherwise it skips the error. func WithHaltOnErrorOnlyForBlocks(blockNames []string) Option { - return func(parser Parser) Parser { - parser.diagnosticsErrorFunc = func(file *File, diags hcl.Diagnostics) (hcl.Diagnostics, error) { - for _, sectionName := range blockNames { - blocks, err := file.Blocks(sectionName, true) + return func(parser *Parser) *Parser { + parser.handleDiagnosticsFunc = appendHandleDiagnosticsFunc(parser.handleDiagnosticsFunc, func(file *File, diags hcl.Diagnostics) (hcl.Diagnostics, error) { + if file == nil || !diags.HasErrors() { + return diags, nil + } + + for _, blockName := range blockNames { + blocks, err := file.Blocks(blockName, true) if err != nil { return nil, err } @@ -47,8 +64,30 @@ func WithHaltOnErrorOnlyForBlocks(blockNames []string) Option { } return nil, nil - } + }) + return parser + } +} +func WithDiagnosticsHandler(fn func(file *hcl.File, diags hcl.Diagnostics) (hcl.Diagnostics, error)) Option { + return func(parser *Parser) *Parser { + parser.handleDiagnosticsFunc = appendHandleDiagnosticsFunc(parser.handleDiagnosticsFunc, func(file *File, diags hcl.Diagnostics) (hcl.Diagnostics, error) { + return fn(file.File, diags) + }) return parser } } + +func appendHandleDiagnosticsFunc(prev, next func(*File, hcl.Diagnostics) (hcl.Diagnostics, error)) func(*File, hcl.Diagnostics) (hcl.Diagnostics, error) { + return func(file *File, diags hcl.Diagnostics) (hcl.Diagnostics, error) { + var err error + + if prev != nil { + if diags, err = prev(file, diags); err != nil { + return diags, err + } + } + + return next(file, diags) + } +} diff --git a/config/hclparse/parser.go b/config/hclparse/parser.go index 728a83548..e0c28644d 100644 --- a/config/hclparse/parser.go +++ b/config/hclparse/parser.go @@ -1,4 +1,4 @@ -// The package wraps `hclparse.Parser` to be able to handle diagnostic errors from one place, see `diagnosticsError(diags hcl.Diagnostics) error` func. +// The package wraps `hclparse.Parser` to be able to handle diagnostic errors from one place, see `handleDiagnostics(diags hcl.Diagnostics) error` func. // This allows us to halt the process only when certain errors occur, such as skipping all errors not related to the `catalog` block. package hclparse @@ -11,13 +11,12 @@ import ( "github.com/gruntwork-io/terragrunt/pkg/log" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclparse" - "github.com/sirupsen/logrus" ) type Parser struct { *hclparse.Parser - logger *logrus.Entry - diagnosticsErrorFunc func(*File, hcl.Diagnostics) (hcl.Diagnostics, error) + loggerFunc func(hcl.Diagnostics) error + handleDiagnosticsFunc func(*File, hcl.Diagnostics) (hcl.Diagnostics, error) fileUpdateHandlerFunc func(*File) error } @@ -29,9 +28,8 @@ func NewParser() *Parser { func (parser *Parser) WithOptions(opts ...Option) *Parser { for _, opt := range opts { - *parser = opt(*parser) + parser = opt(parser) } - return parser } @@ -71,14 +69,37 @@ func (parser *Parser) ParseFromBytes(content []byte, configPath string) (file *F hclFile, diags = parser.ParseHCL(content, configPath) } - if diags.HasErrors() { + file = &File{ + Parser: parser, + File: hclFile, + ConfigPath: configPath, + } + + if err := parser.handleDiagnostics(file, diags); err != nil { log.Warnf("Failed to parse HCL in file %s: %v", configPath, diags) return nil, errors.WithStackTrace(diags) } - return &File{ - Parser: parser, - File: hclFile, - ConfigPath: configPath, - }, nil + return file, nil +} + +func (parser *Parser) handleDiagnostics(file *File, diags hcl.Diagnostics) error { + if len(diags) == 0 { + return nil + } + + if fn := parser.handleDiagnosticsFunc; fn != nil { + var err error + if diags, err = fn(file, diags); err != nil || diags == nil { + return err + } + } + + if fn := parser.loggerFunc; fn != nil { + if err := fn(diags); err != nil { + return err + } + } + + return diags } diff --git a/config/include.go b/config/include.go index 1e9d16c65..b7874c167 100644 --- a/config/include.go +++ b/config/include.go @@ -91,7 +91,7 @@ func parseIncludedConfig(ctx *ParsingContext, includedConfig *IncludeConfig) (*T return PartialParseConfigFile(ctx, includePath, includedConfig) } - return ParseConfigFile(ctx.TerragruntOptions, ctx, includePath, includedConfig) + return ParseConfigFile(ctx, includePath, includedConfig) } // handleInclude merges the included config into the current config depending on the merge strategy specified by the @@ -565,7 +565,6 @@ func mergeInputs(childInputs map[string]interface{}, parentInputs map[string]int for key, value := range childInputs { out[key] = value } - return out } diff --git a/config/locals.go b/config/locals.go index 2ab3e82e4..8a08f2096 100644 --- a/config/locals.go +++ b/config/locals.go @@ -4,6 +4,7 @@ import ( "fmt" "strings" + "github.com/hashicorp/go-multierror" "github.com/hashicorp/hcl/v2" "github.com/zclconf/go-cty/cty" @@ -67,12 +68,18 @@ func evaluateLocalsBlock(ctx *ParsingContext, file *hclparse.File) (map[string]c if len(attrs) > 0 { // This is an error because we couldn't evaluate all locals - ctx.TerragruntOptions.Logger.Errorf("Not all locals could be evaluated:") - for _, local := range attrs { - _, reason := canEvaluateLocals(local.Expr, evaluatedLocals) - ctx.TerragruntOptions.Logger.Errorf("\t- %s [REASON: %s]", local.Name, reason) + ctx.TerragruntOptions.Logger.Debugf("Not all locals could be evaluated:") + var errs *multierror.Error + for _, attr := range attrs { + diags := canEvaluateLocals(attr.Expr, evaluatedLocals) + if err := file.HandleDiagnostics(diags); err != nil { + errs = multierror.Append(errs, err) + } + } + + if err := errs.ErrorOrNil(); err != nil { + return nil, errors.WithStackTrace(CouldNotEvaluateAllLocalsError{Err: err}) } - return nil, errors.WithStackTrace(CouldNotEvaluateAllLocalsError{}) } return evaluatedLocals, nil @@ -90,6 +97,7 @@ func attemptEvaluateLocals( attrs hclparse.Attributes, evaluatedLocals map[string]cty.Value, ) (unevaluatedAttrs hclparse.Attributes, newEvaluatedLocals map[string]cty.Value, evaluated bool, err error) { + localsAsCtyVal, err := convertValuesMapToCtyVal(evaluatedLocals) if err != nil { ctx.TerragruntOptions.Logger.Errorf("Could not convert evaluated locals to the execution ctx to evaluate additional locals in file %s", file.ConfigPath) @@ -113,8 +121,7 @@ func attemptEvaluateLocals( newEvaluatedLocals[key] = val } for _, attr := range attrs { - localEvaluated, _ := canEvaluateLocals(attr.Expr, evaluatedLocals) - if localEvaluated { + if diags := canEvaluateLocals(attr.Expr, evaluatedLocals); !diags.HasErrors() { evaluatedVal, err := attr.Value(evalCtx) if err != nil { return nil, evaluatedLocals, false, err @@ -142,62 +149,51 @@ func attemptEvaluateLocals( // - It has references to other locals that have already been evaluated. // Note that the second return value is a human friendly reason for why the expression can not be evaluated, and is // useful for error reporting. -func canEvaluateLocals(expression hcl.Expression, evaluatedLocals map[string]cty.Value) (bool, string) { - vars := expression.Variables() - if len(vars) == 0 { - // If there are no local variable references, we can evaluate this expression. - return true, "" - } +func canEvaluateLocals(expression hcl.Expression, evaluatedLocals map[string]cty.Value) hcl.Diagnostics { + var diags hcl.Diagnostics - for _, var_ := range vars { - // This should never happen, but if it does, we can't evaluate this expression. - if var_.IsRelative() { - reason := "You've reached an impossible condition and is almost certainly a bug in terragrunt. Please open an issue at github.com/gruntwork-io/terragrunt with this message and the contents of your terragrunt.hcl file that caused this." - return false, reason - } + localVars := expression.Variables() - rootName := var_.RootName() + for _, localVar := range localVars { + var ( + rootName = localVar.RootName() + localName = getLocalName(localVar) + _, hasEvaluated = evaluatedLocals[localName] + detail string + ) - // If the variable is `include`, then we can evaluate it now - if rootName == MetadataInclude { - continue - } + switch { + case localVar.IsRelative(): + // This should never happen, but if it does, we can't evaluate this expression. + detail = "This caused an impossible condition, tnis is almost certainly a bug in Terragrunt. Please open an issue at github.com/gruntwork-io/terragrunt with this message and the contents of your terragrunt.hcl file that caused this." - // We can't evaluate any variable other than `local` - if rootName != "local" { - reason := fmt.Sprintf( - "Can't evaluate expression at %s: you can only reference other local variables here, but it looks like you're referencing something else (%s is not defined)", - expression.Range(), - rootName, - ) - return false, reason - } + case rootName == MetadataInclude: + // If the variable is `include`, then we can evaluate it now + + case rootName != "local": + // We can't evaluate any variable other than `local` + detail = fmt.Sprintf("You can only reference to other local variables here, but it looks like you're referencing something else (%q is not defined)", rootName) + + case localName == "": + // If we can't get any local name, we can't evaluate it. + detail = "This local var name can not be determined." - // If we can't get any local name, we can't evaluate it. - localName := getLocalName(var_) - if localName == "" { - reason := fmt.Sprintf( - "Can't evaluate expression at %s because local var name can not be determined.", - expression.Range(), - ) - return false, reason + case !hasEvaluated: + // If the referenced local isn't evaluated, we can't evaluate this expression. + detail = fmt.Sprintf("The local reference '%s' is not evaluated. Either it is not ready yet in the current pass, or there was an error evaluating it in an earlier stage.", localName) } - // If the referenced local isn't evaluated, we can't evaluate this expression. - _, hasEvaluated := evaluatedLocals[localName] - if !hasEvaluated { - reason := fmt.Sprintf( - "Can't evaluate expression at %s because local reference '%s' is not evaluated. Either it is not ready yet in the current pass, or there was an error evaluating it in an earlier stage.", - expression.Range(), - localName, - ) - return false, reason + if detail != "" { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Can't evaluate expression", + Detail: detail, + Subject: expression.Range().Ptr(), + }) } } - // If we made it this far, this means all the variables referenced are accounted for and we can evaluate this - // expression. - return true, "" + return diags } // getLocalName takes a variable reference encoded as a HCL tree traversal that is rooted at the name `local` and @@ -230,12 +226,18 @@ func getLocalName(traversal hcl.Traversal) string { // Custom Errors Returned by Functions in this Code // ------------------------------------------------ -type CouldNotEvaluateAllLocalsError struct{} +type CouldNotEvaluateAllLocalsError struct { + Err error +} func (err CouldNotEvaluateAllLocalsError) Error() string { return "Could not evaluate all locals in block." } +func (err CouldNotEvaluateAllLocalsError) Unwrap() error { + return err.Err +} + type MaxIterError struct{} func (err MaxIterError) Error() string { diff --git a/config/parsing_context.go b/config/parsing_context.go index 32e523225..2063ddeb7 100644 --- a/config/parsing_context.go +++ b/config/parsing_context.go @@ -8,6 +8,7 @@ import ( "github.com/gruntwork-io/terragrunt/config/hclparse" "github.com/gruntwork-io/terragrunt/options" + "github.com/gruntwork-io/terragrunt/shell" ) // ParsingContext provides various variables that are used throughout all funcs and passed from function to function. @@ -45,6 +46,8 @@ type ParsingContext struct { } func NewParsingContext(ctx context.Context, opts *options.TerragruntOptions) *ParsingContext { + ctx = shell.ContextWithTerraformCommandHook(ctx, nil) + return &ParsingContext{ Context: ctx, TerragruntOptions: opts, @@ -70,3 +73,8 @@ func (ctx ParsingContext) WithTrackInclude(trackInclude *TrackInclude) *ParsingC ctx.TrackInclude = trackInclude return &ctx } + +func (ctx ParsingContext) WithParseOption(parserOptions []hclparse.Option) *ParsingContext { + ctx.ParserOptions = parserOptions + return &ctx +} diff --git a/configstack/errors.go b/configstack/errors.go index 6f0925faf..bac5f4a52 100644 --- a/configstack/errors.go +++ b/configstack/errors.go @@ -1,34 +1,81 @@ package configstack -import "fmt" +import ( + "fmt" + "strings" + + "github.com/gruntwork-io/terragrunt/shell" +) // Custom error types -type UnrecognizedDependency struct { +type UnrecognizedDependencyError struct { ModulePath string DependencyPath string TerragruntConfigPaths []string } -func (err UnrecognizedDependency) Error() string { +func (err UnrecognizedDependencyError) Error() string { return fmt.Sprintf("Module %s specifies %s as a dependency, but that dependency was not one of the ones found while scanning subfolders: %v", err.ModulePath, err.DependencyPath, err.TerragruntConfigPaths) } -type ErrorProcessingModule struct { +type ProcessingModuleError struct { UnderlyingError error ModulePath string HowThisModuleWasFound string } -func (err ErrorProcessingModule) Error() string { +func (err ProcessingModuleError) Error() string { return fmt.Sprintf("Error processing module at '%s'. How this module was found: %s. Underlying error: %v", err.ModulePath, err.HowThisModuleWasFound, err.UnderlyingError) } -type InfiniteRecursion struct { +func (err ProcessingModuleError) Unwrap() error { + return err.UnderlyingError +} + +type InfiniteRecursionError struct { RecursionLevel int Modules map[string]*TerraformModule } -func (err InfiniteRecursion) Error() string { +func (err InfiniteRecursionError) Error() string { return fmt.Sprintf("Hit what seems to be an infinite recursion after going %d levels deep. Please check for a circular dependency! Modules involved: %v", err.RecursionLevel, err.Modules) } + +var NoTerraformModulesFound = fmt.Errorf("Could not find any subfolders with Terragrunt configuration files") + +type DependencyCycleError []string + +func (err DependencyCycleError) Error() string { + return fmt.Sprintf("Found a dependency cycle between modules: %s", strings.Join([]string(err), " -> ")) +} + +type ProcessingModuleDependencyError struct { + Module *TerraformModule + Dependency *TerraformModule + Err error +} + +func (err ProcessingModuleDependencyError) Error() string { + return fmt.Sprintf("Cannot process module %s because one of its dependencies, %s, finished with an error: %s", err.Module, err.Dependency, err.Err) +} + +func (err ProcessingModuleDependencyError) ExitStatus() (int, error) { + if exitCode, err := shell.GetExitCode(err.Err); err == nil { + return exitCode, nil + } + return -1, err +} + +func (err ProcessingModuleDependencyError) Unwrap() error { + return err.Err +} + +type DependencyNotFoundWhileCrossLinkingError struct { + Module *runningModule + Dependency *TerraformModule +} + +func (err DependencyNotFoundWhileCrossLinkingError) Error() string { + return fmt.Sprintf("Module %v specifies a dependency on module %v, but could not find that module while cross-linking dependencies. This is most likely a bug in Terragrunt. Please report it.", err.Module, err.Dependency) +} diff --git a/configstack/graph.go b/configstack/graph.go deleted file mode 100644 index c86d7e331..000000000 --- a/configstack/graph.go +++ /dev/null @@ -1,51 +0,0 @@ -package configstack - -import ( - "github.com/gruntwork-io/go-commons/errors" - "github.com/gruntwork-io/terragrunt/util" -) - -// Check for dependency cycles in the given list of modules and return an error if one is found -func CheckForCycles(modules []*TerraformModule) error { - visitedPaths := []string{} - currentTraversalPaths := []string{} - - for _, module := range modules { - err := checkForCyclesUsingDepthFirstSearch(module, &visitedPaths, ¤tTraversalPaths) - if err != nil { - return err - } - } - - return nil -} - -// Check for cycles using a depth-first-search as described here: -// https://en.wikipedia.org/wiki/Topological_sorting#Depth-first_search -// -// Note that this method uses two lists, visitedPaths, and currentTraversalPaths, to track what nodes have already been -// seen. We need to use lists to maintain ordering so we can show the proper order of paths in a cycle. Of course, a -// list doesn't perform well with repeated contains() and remove() checks, so ideally we'd use an ordered Map (e.g. -// Java's LinkedHashMap), but since Go doesn't have such a data structure built-in, and our lists are going to be very -// small (at most, a few dozen paths), there is no point in worrying about performance. -func checkForCyclesUsingDepthFirstSearch(module *TerraformModule, visitedPaths *[]string, currentTraversalPaths *[]string) error { - if util.ListContainsElement(*visitedPaths, module.Path) { - return nil - } - - if util.ListContainsElement(*currentTraversalPaths, module.Path) { - return errors.WithStackTrace(DependencyCycle(append(*currentTraversalPaths, module.Path))) - } - - *currentTraversalPaths = append(*currentTraversalPaths, module.Path) - for _, dependency := range module.Dependencies { - if err := checkForCyclesUsingDepthFirstSearch(dependency, visitedPaths, currentTraversalPaths); err != nil { - return err - } - } - - *visitedPaths = append(*visitedPaths, module.Path) - *currentTraversalPaths = util.RemoveElementFromList(*currentTraversalPaths, module.Path) - - return nil -} diff --git a/configstack/graph_test.go b/configstack/graph_test.go deleted file mode 100644 index d90000b38..000000000 --- a/configstack/graph_test.go +++ /dev/null @@ -1,87 +0,0 @@ -package configstack - -import ( - "testing" - - "github.com/gruntwork-io/go-commons/errors" - "github.com/stretchr/testify/assert" -) - -func TestCheckForCycles(t *testing.T) { - t.Parallel() - - //////////////////////////////////// - // These modules have no dependencies - //////////////////////////////////// - a := &TerraformModule{Path: "a"} - b := &TerraformModule{Path: "b"} - c := &TerraformModule{Path: "c"} - d := &TerraformModule{Path: "d"} - - //////////////////////////////////// - // These modules have dependencies, but no cycles - //////////////////////////////////// - - // e -> a - e := &TerraformModule{Path: "e", Dependencies: []*TerraformModule{a}} - - // f -> a, b - f := &TerraformModule{Path: "f", Dependencies: []*TerraformModule{a, b}} - - // g -> e -> a - g := &TerraformModule{Path: "g", Dependencies: []*TerraformModule{e}} - - // h -> g -> e -> a - // | / - // --> f -> b - // | - // --> c - h := &TerraformModule{Path: "h", Dependencies: []*TerraformModule{g, f, c}} - - //////////////////////////////////// - // These modules have dependencies and cycles - //////////////////////////////////// - - // i -> i - i := &TerraformModule{Path: "i", Dependencies: []*TerraformModule{}} - i.Dependencies = append(i.Dependencies, i) - - // j -> k -> j - j := &TerraformModule{Path: "j", Dependencies: []*TerraformModule{}} - k := &TerraformModule{Path: "k", Dependencies: []*TerraformModule{j}} - j.Dependencies = append(j.Dependencies, k) - - // l -> m -> n -> o -> l - l := &TerraformModule{Path: "l", Dependencies: []*TerraformModule{}} - o := &TerraformModule{Path: "o", Dependencies: []*TerraformModule{l}} - n := &TerraformModule{Path: "n", Dependencies: []*TerraformModule{o}} - m := &TerraformModule{Path: "m", Dependencies: []*TerraformModule{n}} - l.Dependencies = append(l.Dependencies, m) - - testCases := []struct { - modules []*TerraformModule - expected DependencyCycle - }{ - {[]*TerraformModule{}, nil}, - {[]*TerraformModule{a}, nil}, - {[]*TerraformModule{a, b, c, d}, nil}, - {[]*TerraformModule{a, e}, nil}, - {[]*TerraformModule{a, b, f}, nil}, - {[]*TerraformModule{a, e, g}, nil}, - {[]*TerraformModule{a, b, c, e, f, g, h}, nil}, - {[]*TerraformModule{i}, DependencyCycle([]string{"i", "i"})}, - {[]*TerraformModule{j, k}, DependencyCycle([]string{"j", "k", "j"})}, - {[]*TerraformModule{l, o, n, m}, DependencyCycle([]string{"l", "m", "n", "o", "l"})}, - {[]*TerraformModule{a, l, b, o, n, f, m, h}, DependencyCycle([]string{"l", "m", "n", "o", "l"})}, - } - - for _, testCase := range testCases { - actual := CheckForCycles(testCase.modules) - if testCase.expected == nil { - assert.Nil(t, actual) - } else if assert.NotNil(t, actual, "For modules %v", testCase.modules) { - actualErr := errors.Unwrap(actual).(DependencyCycle) - assert.Equal(t, []string(testCase.expected), []string(actualErr), "For modules %v", testCase.modules) - } - } -} diff --git a/configstack/graphviz.go b/configstack/graphviz.go deleted file mode 100644 index 8dcc38103..000000000 --- a/configstack/graphviz.go +++ /dev/null @@ -1,62 +0,0 @@ -package configstack - -import ( - "fmt" - "io" - "path/filepath" - "strings" - - "github.com/gruntwork-io/go-commons/errors" - - "github.com/gruntwork-io/terragrunt/options" -) - -// WriteDot is used to emit a GraphViz compatible definition -// for a directed graph. It can be used to dump a .dot file. -// This is a similar implementation to terraform's digraph https://github.com/hashicorp/terraform/blob/master/digraph/graphviz.go -// adding some styling to modules that are excluded from the execution in *-all commands -func WriteDot(w io.Writer, terragruntOptions *options.TerragruntOptions, modules []*TerraformModule) error { - - _, err := w.Write([]byte("digraph {\n")) - if err != nil { - return errors.WithStackTrace(err) - } - defer func(w io.Writer, p []byte) { - _, err := w.Write(p) - if err != nil { - terragruntOptions.Logger.Warnf("Failed to close graphviz output: %v", err) - } - }(w, []byte("}\n")) - - // all paths are relative to the TerragruntConfigPath - prefix := filepath.Dir(terragruntOptions.TerragruntConfigPath) + "/" - - for _, source := range modules { - // apply a different coloring for excluded nodes - style := "" - if source.FlagExcluded { - style = "[color=red]" - } - - nodeLine := fmt.Sprintf("\t\"%s\" %s;\n", - strings.TrimPrefix(source.Path, prefix), style) - - _, err := w.Write([]byte(nodeLine)) - if err != nil { - return errors.WithStackTrace(err) - } - - for _, target := range source.Dependencies { - line := fmt.Sprintf("\t\"%s\" -> \"%s\";\n", - strings.TrimPrefix(source.Path, prefix), - strings.TrimPrefix(target.Path, prefix), - ) - _, err := w.Write([]byte(line)) - if err != nil { - return errors.WithStackTrace(err) - } - } - } - - return nil -} diff --git a/configstack/graphviz_test.go b/configstack/graphviz_test.go deleted file mode 100644 index 0edcfffd8..000000000 --- a/configstack/graphviz_test.go +++ /dev/null @@ -1,115 +0,0 @@ -package configstack - -import ( - "bytes" - "strings" - "testing" - - "github.com/gruntwork-io/terragrunt/options" - "github.com/stretchr/testify/assert" -) - -func TestGraph(t *testing.T) { - a := &TerraformModule{Path: "a"} - b := &TerraformModule{Path: "b"} - c := &TerraformModule{Path: "c"} - d := &TerraformModule{Path: "d"} - e := &TerraformModule{Path: "e", Dependencies: []*TerraformModule{a}} - f := &TerraformModule{Path: "f", Dependencies: []*TerraformModule{a, b}} - g := &TerraformModule{Path: "g", Dependencies: []*TerraformModule{e}} - h := &TerraformModule{Path: "h", Dependencies: []*TerraformModule{g, f, c}} - - var stdout bytes.Buffer - terragruntOptions, _ := options.NewTerragruntOptionsForTest("/terragrunt.hcl") - WriteDot(&stdout, terragruntOptions, []*TerraformModule{a, b, c, d, e, f, g, h}) - expected := strings.TrimSpace(` -digraph { - "a" ; - "b" ; - "c" ; - "d" ; - "e" ; - "e" -> "a"; - "f" ; - "f" -> "a"; - "f" -> "b"; - "g" ; - "g" -> "e"; - "h" ; - "h" -> "g"; - "h" -> "f"; - "h" -> "c"; -} -`) - assert.True(t, strings.Contains(stdout.String(), expected)) -} - -func TestGraphTrimPrefix(t *testing.T) { - a := &TerraformModule{Path: "/config/a"} - b := &TerraformModule{Path: "/config/b"} - c := &TerraformModule{Path: "/config/c"} - d := &TerraformModule{Path: "/config/d"} - e := &TerraformModule{Path: "/config/alpha/beta/gamma/e", Dependencies: []*TerraformModule{a}} - f := &TerraformModule{Path: "/config/alpha/beta/gamma/f", Dependencies: []*TerraformModule{a, b}} - g := &TerraformModule{Path: "/config/alpha/g", Dependencies: []*TerraformModule{e}} - h := &TerraformModule{Path: "/config/alpha/beta/h", Dependencies: []*TerraformModule{g, f, c}} - - var stdout bytes.Buffer - terragruntOptions, _ := options.NewTerragruntOptionsWithConfigPath("/config/terragrunt.hcl") - WriteDot(&stdout, terragruntOptions, []*TerraformModule{a, b, c, d, e, f, g, h}) - expected := strings.TrimSpace(` -digraph { - "a" ; - "b" ; - "c" ; - "d" ; - "alpha/beta/gamma/e" ; - "alpha/beta/gamma/e" -> "a"; - "alpha/beta/gamma/f" ; - "alpha/beta/gamma/f" -> "a"; - "alpha/beta/gamma/f" -> "b"; - "alpha/g" ; - "alpha/g" -> "alpha/beta/gamma/e"; - "alpha/beta/h" ; - "alpha/beta/h" -> "alpha/g"; - "alpha/beta/h" -> "alpha/beta/gamma/f"; - "alpha/beta/h" -> "c"; -} -`) - assert.True(t, strings.Contains(stdout.String(), expected)) -} - -func TestGraphFlagExcluded(t *testing.T) { - a := &TerraformModule{Path: "a", FlagExcluded: true} - b := &TerraformModule{Path: "b"} - c := &TerraformModule{Path: "c"} - d := &TerraformModule{Path: "d"} - e := &TerraformModule{Path: "e", Dependencies: []*TerraformModule{a}} - f := &TerraformModule{Path: "f", FlagExcluded: true, Dependencies: []*TerraformModule{a, b}} - g := &TerraformModule{Path: "g", Dependencies: []*TerraformModule{e}} - h := &TerraformModule{Path: "h", Dependencies: []*TerraformModule{g, f, c}} - - var stdout bytes.Buffer - terragruntOptions, _ := options.NewTerragruntOptionsForTest("/terragrunt.hcl") - WriteDot(&stdout, terragruntOptions, []*TerraformModule{a, b, c, d, e, f, g, h}) - expected := strings.TrimSpace(` -digraph { - "a" [color=red]; - "b" ; - "c" ; - "d" ; - "e" ; - "e" -> "a"; - "f" [color=red]; - "f" -> "a"; - "f" -> "b"; - "g" ; - "g" -> "e"; - "h" ; - "h" -> "g"; - "h" -> "f"; - "h" -> "c"; -} -`) - assert.True(t, strings.Contains(stdout.String(), expected)) -} diff --git a/configstack/log.go b/configstack/log.go new file mode 100644 index 000000000..4e48a28d9 --- /dev/null +++ b/configstack/log.go @@ -0,0 +1,44 @@ +package configstack + +import "github.com/sirupsen/logrus" + +// ForceLogLevelHook - log hook which can change log level for messages which contains specific substrings +type ForceLogLevelHook struct { + TriggerLevels []logrus.Level + ForcedLevel logrus.Level +} + +// NewForceLogLevelHook - create default log reduction hook +func NewForceLogLevelHook(forcedLevel logrus.Level) *ForceLogLevelHook { + return &ForceLogLevelHook{ + ForcedLevel: forcedLevel, + TriggerLevels: logrus.AllLevels, + } +} + +// Levels - return log levels on which hook will be triggered +func (hook *ForceLogLevelHook) Levels() []logrus.Level { + return hook.TriggerLevels +} + +// Fire - function invoked against log entries when entry will match loglevel from Levels() +func (hook *ForceLogLevelHook) Fire(entry *logrus.Entry) error { + entry.Level = hook.ForcedLevel + // special formatter to skip printing of log entries since after hook evaluation, entries are printed directly + formatter := LogEntriesDropperFormatter{OriginalFormatter: entry.Logger.Formatter} + entry.Logger.Formatter = &formatter + return nil +} + +// LogEntriesDropperFormatter - custom formatter which will ignore log entries which has lower level than preconfigured in logger +type LogEntriesDropperFormatter struct { + OriginalFormatter logrus.Formatter +} + +// Format - custom entry formatting function which will drop entries with lower level than set in logger +func (formatter *LogEntriesDropperFormatter) Format(entry *logrus.Entry) ([]byte, error) { + if entry.Logger.Level >= entry.Level { + return formatter.OriginalFormatter.Format(entry) + } + return []byte(""), nil +} diff --git a/configstack/log_test.go b/configstack/log_test.go new file mode 100644 index 000000000..7efbe76ef --- /dev/null +++ b/configstack/log_test.go @@ -0,0 +1,50 @@ +package configstack + +import ( + "bytes" + "strings" + "testing" + + "github.com/sirupsen/logrus" + + "github.com/stretchr/testify/assert" +) + +func ptr(str string) *string { + return &str +} + +func TestLogReductionHook(t *testing.T) { + t.Parallel() + var hook = NewForceLogLevelHook(logrus.ErrorLevel) + + stdout := bytes.Buffer{} + + var testLogger = logrus.New() + testLogger.Out = &stdout + testLogger.AddHook(hook) + testLogger.Level = logrus.DebugLevel + + logrus.NewEntry(testLogger).Info("Test tomato") + logrus.NewEntry(testLogger).Error("666 potato 111") + + out := stdout.String() + + var firstLogEntry = "" + var secondLogEntry = "" + + for _, line := range strings.Split(out, "\n") { + if strings.Contains(line, "tomato") { + firstLogEntry = line + continue + } + if strings.Contains(line, "potato") { + secondLogEntry = line + continue + } + } + // check that both entries got logged with error level + assert.Contains(t, firstLogEntry, "level=error") + assert.Contains(t, secondLogEntry, "level=error") + +} diff --git a/configstack/module.go b/configstack/module.go index c9fa4f891..aeeb85c97 100644 --- a/configstack/module.go +++ b/configstack/module.go @@ -4,16 +4,16 @@ import ( "context" "encoding/json" "fmt" + "io" "path/filepath" "sort" "strings" "github.com/gruntwork-io/terragrunt/internal/cache" - "github.com/gruntwork-io/terragrunt/telemetry" + "github.com/gruntwork-io/terragrunt/terraform" "github.com/sirupsen/logrus" - "github.com/gruntwork-io/go-commons/collections" "github.com/gruntwork-io/go-commons/errors" "github.com/gruntwork-io/go-commons/files" "github.com/gruntwork-io/terragrunt/config" @@ -28,7 +28,7 @@ const maxLevelsOfRecursion = 20 // module and the list of other modules that this module depends on type TerraformModule struct { Path string - Dependencies []*TerraformModule + Dependencies TerraformModules Config config.TerragruntConfig TerragruntOptions *options.TerragruntOptions AssumeAlreadyApplied bool @@ -47,115 +47,343 @@ func (module *TerraformModule) String() string { ) } -func (module TerraformModule) MarshalJSON() ([]byte, error) { +func (module *TerraformModule) MarshalJSON() ([]byte, error) { return json.Marshal(module.Path) } -// Go through each of the given Terragrunt configuration files and resolve the module that configuration file represents -// into a TerraformModule struct. Return the list of these TerraformModule structs. -func ResolveTerraformModules(ctx context.Context, terragruntConfigPaths []string, terragruntOptions *options.TerragruntOptions, childTerragruntConfig *config.TerragruntConfig, howThesePathsWereFound string) ([]*TerraformModule, error) { - canonicalTerragruntConfigPaths, err := util.CanonicalPaths(terragruntConfigPaths, ".") - if err != nil { - return nil, err +// Check for cycles using a depth-first-search as described here: +// https://en.wikipedia.org/wiki/Topological_sorting#Depth-first_search +// +// Note that this method uses two lists, visitedPaths, and currentTraversalPaths, to track what nodes have already been +// seen. We need to use lists to maintain ordering so we can show the proper order of paths in a cycle. Of course, a +// list doesn't perform well with repeated contains() and remove() checks, so ideally we'd use an ordered Map (e.g. +// Java's LinkedHashMap), but since Go doesn't have such a data structure built-in, and our lists are going to be very +// small (at most, a few dozen paths), there is no point in worrying about performance. +func (module *TerraformModule) checkForCyclesUsingDepthFirstSearch(visitedPaths *[]string, currentTraversalPaths *[]string) error { + if util.ListContainsElement(*visitedPaths, module.Path) { + return nil } - var modules map[string]*TerraformModule - err = telemetry.Telemetry(ctx, terragruntOptions, "resolve_modules", map[string]interface{}{ - "working_dir": terragruntOptions.WorkingDir, - }, func(childCtx context.Context) error { + if util.ListContainsElement(*currentTraversalPaths, module.Path) { + return errors.WithStackTrace(DependencyCycleError(append(*currentTraversalPaths, module.Path))) + } - result, err := resolveModules(ctx, canonicalTerragruntConfigPaths, terragruntOptions, childTerragruntConfig, howThesePathsWereFound) - if err != nil { + *currentTraversalPaths = append(*currentTraversalPaths, module.Path) + for _, dependency := range module.Dependencies { + if err := dependency.checkForCyclesUsingDepthFirstSearch(visitedPaths, currentTraversalPaths); err != nil { return err } - modules = result - return nil - }) - if err != nil { - return nil, err } - var externalDependencies map[string]*TerraformModule - err = telemetry.Telemetry(ctx, terragruntOptions, "resolve_external_dependencies_for_modules", map[string]interface{}{ - "working_dir": terragruntOptions.WorkingDir, - }, func(childCtx context.Context) error { - result, err := resolveExternalDependenciesForModules(ctx, modules, map[string]*TerraformModule{}, 0, terragruntOptions, childTerragruntConfig) + *visitedPaths = append(*visitedPaths, module.Path) + *currentTraversalPaths = util.RemoveElementFromList(*currentTraversalPaths, module.Path) + + return nil +} + +// planFile - return plan file location, if output folder is set +func (module *TerraformModule) planFile(terragruntOptions *options.TerragruntOptions) string { + planFile := "" + + // set plan file location if output folder is set + planFile = module.outputFile(terragruntOptions) + + planCommand := module.TerragruntOptions.TerraformCommand == terraform.CommandNamePlan || module.TerragruntOptions.TerraformCommand == terraform.CommandNameShow + + // in case if JSON output is enabled, and not specified planFile, save plan in working dir + if planCommand && planFile == "" && module.TerragruntOptions.JsonOutputFolder != "" { + planFile = terraform.TerraformPlanFile + } + return planFile +} + +// outputFile - return plan file location, if output folder is set +func (module *TerraformModule) outputFile(opts *options.TerragruntOptions) string { + planFile := "" + if opts.OutputFolder != "" { + path, _ := filepath.Rel(opts.WorkingDir, module.Path) + dir := filepath.Join(opts.OutputFolder, path) + planFile = filepath.Join(dir, terraform.TerraformPlanFile) + } + return planFile +} + +// outputJsonFile - return plan JSON file location, if JSON output folder is set +func (module *TerraformModule) outputJsonFile(opts *options.TerragruntOptions) string { + jsonPlanFile := "" + if opts.JsonOutputFolder != "" { + path, _ := filepath.Rel(opts.WorkingDir, module.Path) + dir := filepath.Join(opts.JsonOutputFolder, path) + jsonPlanFile = filepath.Join(dir, terraform.TerraformPlanJsonFile) + } + return jsonPlanFile +} + +// findModuleInPath returns true if a module is located under one of the target directories +func (module *TerraformModule) findModuleInPath(targetDirs []string) bool { + for _, targetDir := range targetDirs { + if module.Path == targetDir { + return true + } + } + return false +} + +// Confirm with the user whether they want Terragrunt to assume the given dependency of the given module is already +// applied. If the user selects "yes", then Terragrunt will apply that module as well. +// Note that we skip the prompt for `run-all destroy` calls. Given the destructive and irreversible nature of destroy, we don't +// want to provide any risk to the user of accidentally destroying an external dependency unless explicitly included +// with the --terragrunt-include-external-dependencies or --terragrunt-include-dir flags. +func (module *TerraformModule) confirmShouldApplyExternalDependency(dependency *TerraformModule, terragruntOptions *options.TerragruntOptions) (bool, error) { + if terragruntOptions.IncludeExternalDependencies { + terragruntOptions.Logger.Debugf("The --terragrunt-include-external-dependencies flag is set, so automatically including all external dependencies, and will run this command against module %s, which is a dependency of module %s.", dependency.Path, module.Path) + return true, nil + } + + if terragruntOptions.NonInteractive { + terragruntOptions.Logger.Debugf("The --non-interactive flag is set. To avoid accidentally affecting external dependencies with a run-all command, will not run this command against module %s, which is a dependency of module %s.", dependency.Path, module.Path) + return false, nil + } + + stackCmd := terragruntOptions.TerraformCommand + if stackCmd == "destroy" { + terragruntOptions.Logger.Debugf("run-all command called with destroy. To avoid accidentally having destructive effects on external dependencies with run-all command, will not run this command against module %s, which is a dependency of module %s.", dependency.Path, module.Path) + return false, nil + } + + prompt := fmt.Sprintf("Module: \t\t %s\nExternal dependency: \t %s\nShould Terragrunt apply the external dependency?", module.Path, dependency.Path) + return shell.PromptUserForYesNo(prompt, terragruntOptions) +} + +// Get the list of modules this module depends on +func (module *TerraformModule) getDependenciesForModule(modulesMap TerraformModulesMap, terragruntConfigPaths []string) (TerraformModules, error) { + dependencies := TerraformModules{} + + if module.Config.Dependencies == nil || len(module.Config.Dependencies.Paths) == 0 { + return dependencies, nil + } + + for _, dependencyPath := range module.Config.Dependencies.Paths { + dependencyModulePath, err := util.CanonicalPath(dependencyPath, module.Path) if err != nil { - return err + return dependencies, nil } - externalDependencies = result - return nil - }) + + if files.FileExists(dependencyModulePath) && !files.IsDir(dependencyModulePath) { + dependencyModulePath = filepath.Dir(dependencyModulePath) + } + + dependencyModule, foundModule := modulesMap[dependencyModulePath] + if !foundModule { + err := UnrecognizedDependencyError{ + ModulePath: module.Path, + DependencyPath: dependencyPath, + TerragruntConfigPaths: terragruntConfigPaths, + } + return dependencies, errors.WithStackTrace(err) + } + dependencies = append(dependencies, dependencyModule) + } + + return dependencies, nil +} + +type TerraformModules []*TerraformModule + +// FindWhereWorkingDirIsIncluded - find where working directory is included, flow: +// 1. Find root git top level directory and build list of modules +// 2. Iterate over includes from terragruntOptions if git top level directory detection failed +// 3. Filter found module only items which has in dependencies working directory +func FindWhereWorkingDirIsIncluded(ctx context.Context, terragruntOptions *options.TerragruntOptions, terragruntConfig *config.TerragruntConfig) TerraformModules { + var pathsToCheck []string + var matchedModulesMap = make(TerraformModulesMap) + + if gitTopLevelDir, err := shell.GitTopLevelDir(ctx, terragruntOptions, terragruntOptions.WorkingDir); err == nil { + pathsToCheck = append(pathsToCheck, gitTopLevelDir) + } else { + // detection failed, trying to use include directories as source for stacks + uniquePaths := make(map[string]bool) + for _, includePath := range terragruntConfig.ProcessedIncludes { + uniquePaths[filepath.Dir(includePath.Path)] = true + } + for path := range uniquePaths { + pathsToCheck = append(pathsToCheck, path) + } + } + + for _, dir := range pathsToCheck { // iterate over detected paths, build stacks and filter modules by working dir + dir += filepath.FromSlash("/") + cfgOptions, err := options.NewTerragruntOptionsWithConfigPath(dir) + if err != nil { + terragruntOptions.Logger.Debugf("Failed to build terragrunt options from %s %v", dir, err) + continue + } + + cfgOptions.Env = terragruntOptions.Env + cfgOptions.LogLevel = terragruntOptions.LogLevel + cfgOptions.OriginalTerragruntConfigPath = terragruntOptions.OriginalTerragruntConfigPath + cfgOptions.TerraformCommand = terragruntOptions.TerraformCommand + cfgOptions.NonInteractive = true + + var hook = NewForceLogLevelHook(logrus.DebugLevel) + cfgOptions.Logger.Logger.AddHook(hook) + + // build stack from config directory + stack, err := FindStackInSubfolders(ctx, cfgOptions, WithChildTerragruntConfig(terragruntConfig)) + if err != nil { + // log error as debug since in some cases stack building may fail because parent files can be designed + // to work with relative paths from downstream modules + terragruntOptions.Logger.Debugf("Failed to build module stack %v", err) + continue + } + + dependentModules := stack.ListStackDependentModules() + deps, found := dependentModules[terragruntOptions.WorkingDir] + if found { + for _, module := range stack.Modules { + for _, dep := range deps { + if dep == module.Path { + matchedModulesMap[module.Path] = module + break + } + } + } + } + } + + // extract modules as list + var matchedModules TerraformModules + for _, module := range matchedModulesMap { + matchedModules = append(matchedModules, module) + } + + return matchedModules +} + +// WriteDot is used to emit a GraphViz compatible definition +// for a directed graph. It can be used to dump a .dot file. +// This is a similar implementation to terraform's digraph https://github.com/hashicorp/terraform/blob/master/digraph/graphviz.go +// adding some styling to modules that are excluded from the execution in *-all commands +func (modules TerraformModules) WriteDot(w io.Writer, terragruntOptions *options.TerragruntOptions) error { + _, err := w.Write([]byte("digraph {\n")) if err != nil { - return nil, err + return errors.WithStackTrace(err) } + defer func(w io.Writer, p []byte) { + _, err := w.Write(p) + if err != nil { + terragruntOptions.Logger.Warnf("Failed to close graphviz output: %v", err) + } + }(w, []byte("}\n")) + + // all paths are relative to the TerragruntConfigPath + prefix := filepath.Dir(terragruntOptions.TerragruntConfigPath) + "/" + + for _, source := range modules { + // apply a different coloring for excluded nodes + style := "" + if source.FlagExcluded { + style = "[color=red]" + } - var crossLinkedModules []*TerraformModule - err = telemetry.Telemetry(ctx, terragruntOptions, "crosslink_dependencies", map[string]interface{}{ - "working_dir": terragruntOptions.WorkingDir, - }, func(childCtx context.Context) error { - result, err := crosslinkDependencies(mergeMaps(modules, externalDependencies), canonicalTerragruntConfigPaths) + nodeLine := fmt.Sprintf("\t\"%s\" %s;\n", + strings.TrimPrefix(source.Path, prefix), style) + + _, err := w.Write([]byte(nodeLine)) if err != nil { - return err + return errors.WithStackTrace(err) } - crossLinkedModules = result - return nil - }) + + for _, target := range source.Dependencies { + line := fmt.Sprintf("\t\"%s\" -> \"%s\";\n", + strings.TrimPrefix(source.Path, prefix), + strings.TrimPrefix(target.Path, prefix), + ) + _, err := w.Write([]byte(line)) + if err != nil { + return errors.WithStackTrace(err) + } + } + } + + return nil +} + +// Run the given map of module path to runningModule. To "run" a module, execute the RunTerragrunt command in its +// TerragruntOptions object. The modules will be executed in an order determined by their inter-dependencies, using +// as much concurrency as possible. +func (modules TerraformModules) RunModules(ctx context.Context, opts *options.TerragruntOptions, parallelism int) error { + runningModules, err := modules.toRunningModules(NormalOrder) if err != nil { - return nil, err + return err } + return runningModules.runModules(ctx, opts, parallelism) +} - var includedModules []*TerraformModule - err = telemetry.Telemetry(ctx, terragruntOptions, "flag_included_dirs", map[string]interface{}{ - "working_dir": terragruntOptions.WorkingDir, - }, func(childCtx context.Context) error { - includedModules = flagIncludedDirs(crossLinkedModules, terragruntOptions) - return nil - }) +// Run the given map of module path to runningModule. To "run" a module, execute the RunTerragrunt command in its +// TerragruntOptions object. The modules will be executed in the reverse order of their inter-dependencies, using +// as much concurrency as possible. +func (modules TerraformModules) RunModulesReverseOrder(ctx context.Context, opts *options.TerragruntOptions, parallelism int) error { + runningModules, err := modules.toRunningModules(ReverseOrder) if err != nil { - return nil, err + return err } + return runningModules.runModules(ctx, opts, parallelism) +} - var includedModulesWithExcluded []*TerraformModule - err = telemetry.Telemetry(ctx, terragruntOptions, "flag_excluded_dirs", map[string]interface{}{ - "working_dir": terragruntOptions.WorkingDir, - }, func(childCtx context.Context) error { - includedModulesWithExcluded = flagExcludedDirs(includedModules, terragruntOptions) - return nil - }) +// Run the given map of module path to runningModule. To "run" a module, execute the RunTerragrunt command in its +// TerragruntOptions object. The modules will be executed without caring for inter-dependencies. +func (modules TerraformModules) RunModulesIgnoreOrder(ctx context.Context, opts *options.TerragruntOptions, parallelism int) error { + runningModules, err := modules.toRunningModules(IgnoreOrder) if err != nil { - return nil, err + return err + } + return runningModules.runModules(ctx, opts, parallelism) +} + +// Convert the list of modules to a map from module path to a runningModule struct. This struct contains information +// about executing the module, such as whether it has finished running or not and any errors that happened. Note that +// this does NOT actually run the module. For that, see the RunModules method. +func (modules TerraformModules) toRunningModules(dependencyOrder DependencyOrder) (runningModules, error) { + runningModules := runningModules{} + for _, module := range modules { + runningModules[module.Path] = newRunningModule(module) } - var finalModules []*TerraformModule - err = telemetry.Telemetry(ctx, terragruntOptions, "flag_modules_that_dont_include", map[string]interface{}{ - "working_dir": terragruntOptions.WorkingDir, - }, func(childCtx context.Context) error { - result, err := flagModulesThatDontInclude(includedModulesWithExcluded, terragruntOptions) + crossLinkedModules, err := runningModules.crossLinkDependencies(dependencyOrder) + if err != nil { + return crossLinkedModules, err + } + + return crossLinkedModules.removeFlagExcluded(), nil +} + +// Check for dependency cycles in the given list of modules and return an error if one is found +func (modules TerraformModules) CheckForCycles() error { + visitedPaths := []string{} + currentTraversalPaths := []string{} + + for _, module := range modules { + err := module.checkForCyclesUsingDepthFirstSearch(&visitedPaths, ¤tTraversalPaths) if err != nil { return err } - finalModules = result - return nil - }) - if err != nil { - return nil, err } - return finalModules, nil + return nil } // flagExcludedDirs iterates over a module slice and flags all entries as excluded, which should be ignored via the terragrunt-exclude-dir CLI flag. -func flagExcludedDirs(modules []*TerraformModule, terragruntOptions *options.TerragruntOptions) []*TerraformModule { +func (modules TerraformModules) flagExcludedDirs(terragruntOptions *options.TerragruntOptions) TerraformModules { for _, module := range modules { - if findModuleInPath(module, terragruntOptions.ExcludeDirs) { + if module.findModuleInPath(terragruntOptions.ExcludeDirs) { // Mark module itself as excluded module.FlagExcluded = true } // Mark all affected dependencies as excluded for _, dependency := range module.Dependencies { - if findModuleInPath(dependency, terragruntOptions.ExcludeDirs) { + if dependency.findModuleInPath(terragruntOptions.ExcludeDirs) { dependency.FlagExcluded = true } } @@ -165,20 +393,19 @@ func flagExcludedDirs(modules []*TerraformModule, terragruntOptions *options.Ter } // flagIncludedDirs iterates over a module slice and flags all entries not in the list specified via the terragrunt-include-dir CLI flag as excluded. -func flagIncludedDirs(modules []*TerraformModule, terragruntOptions *options.TerragruntOptions) []*TerraformModule { - +func (modules TerraformModules) flagIncludedDirs(terragruntOptions *options.TerragruntOptions) TerraformModules { // If no IncludeDirs is specified return the modules list instantly if len(terragruntOptions.IncludeDirs) == 0 { // If we aren't given any include directories, but are given the strict include flag, // return no modules. if terragruntOptions.StrictInclude { - return []*TerraformModule{} + return TerraformModules{} } return modules } for _, module := range modules { - if findModuleInPath(module, terragruntOptions.IncludeDirs) { + if module.findModuleInPath(terragruntOptions.IncludeDirs) { module.FlagExcluded = false } else { module.FlagExcluded = true @@ -199,21 +426,10 @@ func flagIncludedDirs(modules []*TerraformModule, terragruntOptions *options.Ter return modules } -// findModuleInPath returns true if a module is located under one of the target directories -func findModuleInPath(module *TerraformModule, targetDirs []string) bool { - for _, targetDir := range targetDirs { - if module.Path == targetDir { - return true - } - } - return false -} - // flagModulesThatDontInclude iterates over a module slice and flags all modules that don't include at least one file in // the specified include list on the TerragruntOptions ModulesThatInclude attribute. Flagged modules will be filtered // out of the set. -func flagModulesThatDontInclude(modules []*TerraformModule, terragruntOptions *options.TerragruntOptions) ([]*TerraformModule, error) { - +func (modules TerraformModules) flagModulesThatDontInclude(terragruntOptions *options.TerragruntOptions) (TerraformModules, error) { // If no ModulesThatInclude is specified return the modules list instantly if len(terragruntOptions.ModulesThatInclude) == 0 { return modules, nil @@ -274,278 +490,20 @@ func flagModulesThatDontInclude(modules []*TerraformModule, terragruntOptions *o return modules, nil } -// Go through each of the given Terragrunt configuration files and resolve the module that configuration file represents -// into a TerraformModule struct. Note that this method will NOT fill in the Dependencies field of the TerraformModule -// struct (see the crosslinkDependencies method for that). Return a map from module path to TerraformModule struct. -func resolveModules(ctx context.Context, canonicalTerragruntConfigPaths []string, terragruntOptions *options.TerragruntOptions, childTerragruntConfig *config.TerragruntConfig, howTheseModulesWereFound string) (map[string]*TerraformModule, error) { - moduleMap := map[string]*TerraformModule{} - for _, terragruntConfigPath := range canonicalTerragruntConfigPaths { - var module *TerraformModule - err := telemetry.Telemetry(ctx, terragruntOptions, "resolve_terraform_module", map[string]interface{}{ - "config_path": terragruntConfigPath, - "working_dir": terragruntOptions.WorkingDir, - }, func(childCtx context.Context) error { - m, err := resolveTerraformModule(terragruntConfigPath, moduleMap, terragruntOptions, childTerragruntConfig, howTheseModulesWereFound) - if err != nil { - return err - } - module = m - return nil - }) - if err != nil { - return moduleMap, err - } - if module != nil { - moduleMap[module.Path] = module - var dependencies map[string]*TerraformModule - err := telemetry.Telemetry(ctx, terragruntOptions, "resolve_dependencies_for_module", map[string]interface{}{ - "config_path": terragruntConfigPath, - "working_dir": terragruntOptions.WorkingDir, - "module_path": module.Path, - }, func(childCtx context.Context) error { - deps, err := resolveDependenciesForModule(ctx, module, moduleMap, terragruntOptions, childTerragruntConfig, true) - if err != nil { - return err - } - dependencies = deps - return nil - }) - if err != nil { - return moduleMap, err - } - moduleMap = collections.MergeMaps(moduleMap, dependencies) - } - } +var existingModules = cache.NewCache[*TerraformModulesMap]() - return moduleMap, nil -} - -// Create a TerraformModule struct for the Terraform module specified by the given Terragrunt configuration file path. -// Note that this method will NOT fill in the Dependencies field of the TerraformModule struct (see the -// crosslinkDependencies method for that). -func resolveTerraformModule(terragruntConfigPath string, moduleMap map[string]*TerraformModule, terragruntOptions *options.TerragruntOptions, childTerragruntConfig *config.TerragruntConfig, howThisModuleWasFound string) (*TerraformModule, error) { - modulePath, err := util.CanonicalPath(filepath.Dir(terragruntConfigPath), ".") - if err != nil { - return nil, err - } - - if _, ok := moduleMap[modulePath]; ok { - return nil, nil - } - - // Clone the options struct so we don't modify the original one. This is especially important as run-all operations - // happen concurrently. - opts := terragruntOptions.Clone(terragruntConfigPath) - - // We need to reset the original path for each module. Otherwise, this path will be set to wherever you ran run-all - // from, which is not what any of the modules will want. - opts.OriginalTerragruntConfigPath = terragruntConfigPath - - // If `childTerragruntConfig.ProcessedIncludes` contains the path `terragruntConfigPath`, then this is a parent config - // which implies that `TerragruntConfigPath` must refer to a child configuration file, and the defined `IncludeConfig` must contain the path to the file itself - // for the built-in functions `read-terragrunt-config()`, `path_relative_to_include()` to work correctly. - var includeConfig *config.IncludeConfig - if childTerragruntConfig != nil && childTerragruntConfig.ProcessedIncludes.ContainsPath(terragruntConfigPath) { - includeConfig = &config.IncludeConfig{Path: terragruntConfigPath} - opts.TerragruntConfigPath = terragruntOptions.OriginalTerragruntConfigPath - } - - if collections.ListContainsElement(opts.ExcludeDirs, modulePath) { - // module is excluded - return &TerraformModule{Path: modulePath, TerragruntOptions: opts, FlagExcluded: true}, nil - } - - configContext := config.NewParsingContext(context.Background(), opts).WithDecodeList( - // Need for initializing the modules - config.TerraformSource, - - // Need for parsing out the dependencies - config.DependenciesBlock, - config.DependencyBlock, - ) - - // We only partially parse the config, only using the pieces that we need in this section. This config will be fully - // parsed at a later stage right before the action is run. This is to delay interpolation of functions until right - // before we call out to terraform. - terragruntConfig, err := config.PartialParseConfigFile( - configContext, - terragruntConfigPath, - includeConfig, - ) - if err != nil { - return nil, errors.WithStackTrace(ErrorProcessingModule{UnderlyingError: err, HowThisModuleWasFound: howThisModuleWasFound, ModulePath: terragruntConfigPath}) - } - - terragruntSource, err := config.GetTerragruntSourceForModule(terragruntOptions.Source, modulePath, terragruntConfig) - if err != nil { - return nil, err - } - opts.Source = terragruntSource - - _, defaultDownloadDir, err := options.DefaultWorkingAndDownloadDirs(terragruntOptions.TerragruntConfigPath) - if err != nil { - return nil, err - } - - // If we're using the default download directory, put it into the same folder as the Terragrunt configuration file. - // If we're not using the default, then the user has specified a custom download directory, and we leave it as-is. - if terragruntOptions.DownloadDir == defaultDownloadDir { - _, downloadDir, err := options.DefaultWorkingAndDownloadDirs(terragruntConfigPath) - if err != nil { - return nil, err - } - terragruntOptions.Logger.Debugf("Setting download directory for module %s to %s", modulePath, downloadDir) - opts.DownloadDir = downloadDir - } - - // Fix for https://github.com/gruntwork-io/terragrunt/issues/208 - matches, err := filepath.Glob(filepath.Join(filepath.Dir(terragruntConfigPath), "*.tf")) - if err != nil { - return nil, err - } - if (terragruntConfig.Terraform == nil || terragruntConfig.Terraform.Source == nil || *terragruntConfig.Terraform.Source == "") && matches == nil { - terragruntOptions.Logger.Debugf("Module %s does not have an associated terraform configuration and will be skipped.", filepath.Dir(terragruntConfigPath)) - return nil, nil - } - - if opts.IncludeModulePrefix { - opts.OutputPrefix = fmt.Sprintf("[%v] ", modulePath) - } - - return &TerraformModule{Path: modulePath, Config: *terragruntConfig, TerragruntOptions: opts}, nil -} - -// Look through the dependencies of the modules in the given map and resolve the "external" dependency paths listed in -// each modules config (i.e. those dependencies not in the given list of Terragrunt config canonical file paths). -// These external dependencies are outside of the current working directory, which means they may not be part of the -// environment the user is trying to apply-all or destroy-all. Therefore, this method also confirms whether the user wants -// to actually apply those dependencies or just assume they are already applied. Note that this method will NOT fill in -// the Dependencies field of the TerraformModule struct (see the crosslinkDependencies method for that). -func resolveExternalDependenciesForModules(ctx context.Context, moduleMap map[string]*TerraformModule, modulesAlreadyProcessed map[string]*TerraformModule, recursionLevel int, terragruntOptions *options.TerragruntOptions, childTerragruntConfig *config.TerragruntConfig) (map[string]*TerraformModule, error) { - allExternalDependencies := map[string]*TerraformModule{} - modulesToSkip := mergeMaps(moduleMap, modulesAlreadyProcessed) - - // Simple protection from circular dependencies causing a Stack Overflow due to infinite recursion - if recursionLevel > maxLevelsOfRecursion { - return allExternalDependencies, errors.WithStackTrace(InfiniteRecursion{RecursionLevel: maxLevelsOfRecursion, Modules: modulesToSkip}) - } - - sortedKeys := getSortedKeys(moduleMap) - for _, key := range sortedKeys { - module := moduleMap[key] - externalDependencies, err := resolveDependenciesForModule(ctx, module, modulesToSkip, terragruntOptions, childTerragruntConfig, false) - if err != nil { - return externalDependencies, err - } - - for _, externalDependency := range externalDependencies { - if _, alreadyFound := modulesToSkip[externalDependency.Path]; alreadyFound { - continue - } - - shouldApply := false - if !terragruntOptions.IgnoreExternalDependencies { - shouldApply, err = confirmShouldApplyExternalDependency(module, externalDependency, terragruntOptions) - if err != nil { - return externalDependencies, err - } - } - - externalDependency.AssumeAlreadyApplied = !shouldApply - allExternalDependencies[externalDependency.Path] = externalDependency - } - } - - if len(allExternalDependencies) > 0 { - recursiveDependencies, err := resolveExternalDependenciesForModules(ctx, allExternalDependencies, moduleMap, recursionLevel+1, terragruntOptions, childTerragruntConfig) - if err != nil { - return allExternalDependencies, err - } - return mergeMaps(allExternalDependencies, recursiveDependencies), nil - } - - return allExternalDependencies, nil -} - -var existingModules = cache.NewCache[*map[string]*TerraformModule]() - -// resolveDependenciesForModule looks through the dependencies of the given module and resolve the dependency paths listed in the module's config. -// If `skipExternal` is true, the func returns only dependencies that are inside of the current working directory, which means they are part of the environment the -// user is trying to apply-all or destroy-all. Note that this method will NOT fill in the Dependencies field of the TerraformModule struct (see the crosslinkDependencies method for that). -func resolveDependenciesForModule(ctx context.Context, module *TerraformModule, moduleMap map[string]*TerraformModule, terragruntOptions *options.TerragruntOptions, chilTerragruntConfig *config.TerragruntConfig, skipExternal bool) (map[string]*TerraformModule, error) { - if module.Config.Dependencies == nil || len(module.Config.Dependencies.Paths) == 0 { - return map[string]*TerraformModule{}, nil - } - - key := fmt.Sprintf("%s-%s-%v-%v", module.Path, terragruntOptions.WorkingDir, skipExternal, terragruntOptions.TerraformCommand) - if value, ok := existingModules.Get(key); ok { - return *value, nil - } - - externalTerragruntConfigPaths := []string{} - for _, dependency := range module.Config.Dependencies.Paths { - dependencyPath, err := util.CanonicalPath(dependency, module.Path) - if err != nil { - return map[string]*TerraformModule{}, err - } - - if skipExternal && !util.HasPathPrefix(dependencyPath, terragruntOptions.WorkingDir) { - continue - } - - terragruntConfigPath := config.GetDefaultConfigPath(dependencyPath) - - if _, alreadyContainsModule := moduleMap[dependencyPath]; !alreadyContainsModule { - externalTerragruntConfigPaths = append(externalTerragruntConfigPaths, terragruntConfigPath) - } - } - - howThesePathsWereFound := fmt.Sprintf("dependency of module at '%s'", module.Path) - result, err := resolveModules(ctx, externalTerragruntConfigPaths, terragruntOptions, chilTerragruntConfig, howThesePathsWereFound) - if err != nil { - return nil, err - } - - existingModules.Put(key, &result) - return result, nil -} - -// Confirm with the user whether they want Terragrunt to assume the given dependency of the given module is already -// applied. If the user selects "yes", then Terragrunt will apply that module as well. -// Note that we skip the prompt for `run-all destroy` calls. Given the destructive and irreversible nature of destroy, we don't -// want to provide any risk to the user of accidentally destroying an external dependency unless explicitly included -// with the --terragrunt-include-external-dependencies or --terragrunt-include-dir flags. -func confirmShouldApplyExternalDependency(module *TerraformModule, dependency *TerraformModule, terragruntOptions *options.TerragruntOptions) (bool, error) { - if terragruntOptions.IncludeExternalDependencies { - terragruntOptions.Logger.Debugf("The --terragrunt-include-external-dependencies flag is set, so automatically including all external dependencies, and will run this command against module %s, which is a dependency of module %s.", dependency.Path, module.Path) - return true, nil - } - - if terragruntOptions.NonInteractive { - terragruntOptions.Logger.Debugf("The --non-interactive flag is set. To avoid accidentally affecting external dependencies with a run-all command, will not run this command against module %s, which is a dependency of module %s.", dependency.Path, module.Path) - return false, nil - } - - stackCmd := terragruntOptions.TerraformCommand - if stackCmd == "destroy" { - terragruntOptions.Logger.Debugf("run-all command called with destroy. To avoid accidentally having destructive effects on external dependencies with run-all command, will not run this command against module %s, which is a dependency of module %s.", dependency.Path, module.Path) - return false, nil - } - - prompt := fmt.Sprintf("Module: \t\t %s\nExternal dependency: \t %s\nShould Terragrunt apply the external dependency?", module.Path, dependency.Path) - return shell.PromptUserForYesNo(prompt, terragruntOptions) -} +type TerraformModulesMap map[string]*TerraformModule // Merge the given external dependencies into the given map of modules if those dependencies aren't already in the // modules map -func mergeMaps(modules map[string]*TerraformModule, externalDependencies map[string]*TerraformModule) map[string]*TerraformModule { - out := map[string]*TerraformModule{} +func (modulesMap TerraformModulesMap) mergeMaps(externalDependencies TerraformModulesMap) TerraformModulesMap { + out := TerraformModulesMap{} for key, value := range externalDependencies { out[key] = value } - for key, value := range modules { + for key, value := range modulesMap { out[key] = value } @@ -554,13 +512,13 @@ func mergeMaps(modules map[string]*TerraformModule, externalDependencies map[str // Go through each module in the given map and cross-link its dependencies to the other modules in that same map. If // a dependency is referenced that is not in the given map, return an error. -func crosslinkDependencies(moduleMap map[string]*TerraformModule, canonicalTerragruntConfigPaths []string) ([]*TerraformModule, error) { - modules := []*TerraformModule{} +func (modulesMap TerraformModulesMap) crosslinkDependencies(canonicalTerragruntConfigPaths []string) (TerraformModules, error) { + modules := TerraformModules{} - keys := getSortedKeys(moduleMap) + keys := modulesMap.getSortedKeys() for _, key := range keys { - module := moduleMap[key] - dependencies, err := getDependenciesForModule(module, moduleMap, canonicalTerragruntConfigPaths) + module := modulesMap[key] + dependencies, err := module.getDependenciesForModule(modulesMap, canonicalTerragruntConfigPaths) if err != nil { return modules, err } @@ -572,44 +530,11 @@ func crosslinkDependencies(moduleMap map[string]*TerraformModule, canonicalTerra return modules, nil } -// Get the list of modules this module depends on -func getDependenciesForModule(module *TerraformModule, moduleMap map[string]*TerraformModule, terragruntConfigPaths []string) ([]*TerraformModule, error) { - dependencies := []*TerraformModule{} - - if module.Config.Dependencies == nil || len(module.Config.Dependencies.Paths) == 0 { - return dependencies, nil - } - - for _, dependencyPath := range module.Config.Dependencies.Paths { - dependencyModulePath, err := util.CanonicalPath(dependencyPath, module.Path) - if err != nil { - return dependencies, nil - } - - if files.FileExists(dependencyModulePath) && !files.IsDir(dependencyModulePath) { - dependencyModulePath = filepath.Dir(dependencyModulePath) - } - - dependencyModule, foundModule := moduleMap[dependencyModulePath] - if !foundModule { - err := UnrecognizedDependency{ - ModulePath: module.Path, - DependencyPath: dependencyPath, - TerragruntConfigPaths: terragruntConfigPaths, - } - return dependencies, errors.WithStackTrace(err) - } - dependencies = append(dependencies, dependencyModule) - } - - return dependencies, nil -} - // Return the keys for the given map in sorted order. This is used to ensure we always iterate over maps of modules // in a consistent order (Go does not guarantee iteration order for maps, and usually makes it random) -func getSortedKeys(modules map[string]*TerraformModule) []string { +func (modulesMap TerraformModulesMap) getSortedKeys() []string { keys := []string{} - for key := range modules { + for key := range modulesMap { keys = append(keys, key) } @@ -617,175 +542,3 @@ func getSortedKeys(modules map[string]*TerraformModule) []string { return keys } - -// FindWhereWorkingDirIsIncluded - find where working directory is included, flow: -// 1. Find root git top level directory and build list of modules -// 2. Iterate over includes from terragruntOptions if git top level directory detection failed -// 3. Filter found module only items which has in dependencies working directory -func FindWhereWorkingDirIsIncluded(ctx context.Context, terragruntOptions *options.TerragruntOptions, terragruntConfig *config.TerragruntConfig) []*TerraformModule { - var pathsToCheck []string - var matchedModulesMap = make(map[string]*TerraformModule) - - if gitTopLevelDir, err := shell.GitTopLevelDir(ctx, terragruntOptions, terragruntOptions.WorkingDir); err == nil { - pathsToCheck = append(pathsToCheck, gitTopLevelDir) - } else { - // detection failed, trying to use include directories as source for stacks - uniquePaths := make(map[string]bool) - for _, includePath := range terragruntConfig.ProcessedIncludes { - uniquePaths[filepath.Dir(includePath.Path)] = true - } - for path := range uniquePaths { - pathsToCheck = append(pathsToCheck, path) - } - } - - for _, dir := range pathsToCheck { // iterate over detected paths, build stacks and filter modules by working dir - dir += filepath.FromSlash("/") - cfgOptions, err := options.NewTerragruntOptionsWithConfigPath(dir) - if err != nil { - terragruntOptions.Logger.Debugf("Failed to build terragrunt options from %s %v", dir, err) - continue - } - - cfgOptions.Env = terragruntOptions.Env - cfgOptions.LogLevel = terragruntOptions.LogLevel - cfgOptions.OriginalTerragruntConfigPath = terragruntOptions.OriginalTerragruntConfigPath - cfgOptions.TerraformCommand = terragruntOptions.TerraformCommand - cfgOptions.NonInteractive = true - - var hook = NewForceLogLevelHook(logrus.DebugLevel) - cfgOptions.Logger.Logger.AddHook(hook) - - // build stack from config directory - stack, err := FindStackInSubfolders(ctx, cfgOptions, terragruntConfig) - if err != nil { - // log error as debug since in some cases stack building may fail because parent files can be designed - // to work with relative paths from downstream modules - terragruntOptions.Logger.Debugf("Failed to build module stack %v", err) - continue - } - - dependentModules := ListStackDependentModules(stack) - deps, found := dependentModules[terragruntOptions.WorkingDir] - if found { - for _, module := range stack.Modules { - for _, dep := range deps { - if dep == module.Path { - matchedModulesMap[module.Path] = module - break - } - } - } - } - } - - // extract modules as list - var matchedModules []*TerraformModule - for _, module := range matchedModulesMap { - matchedModules = append(matchedModules, module) - } - - return matchedModules -} - -// ForceLogLevelHook - log hook which can change log level for messages which contains specific substrings -type ForceLogLevelHook struct { - TriggerLevels []logrus.Level - ForcedLevel logrus.Level -} - -// NewForceLogLevelHook - create default log reduction hook -func NewForceLogLevelHook(forcedLevel logrus.Level) *ForceLogLevelHook { - return &ForceLogLevelHook{ - ForcedLevel: forcedLevel, - TriggerLevels: logrus.AllLevels, - } -} - -// Levels - return log levels on which hook will be triggered -func (hook *ForceLogLevelHook) Levels() []logrus.Level { - return hook.TriggerLevels -} - -// Fire - function invoked against log entries when entry will match loglevel from Levels() -func (hook *ForceLogLevelHook) Fire(entry *logrus.Entry) error { - entry.Level = hook.ForcedLevel - // special formatter to skip printing of log entries since after hook evaluation, entries are printed directly - formatter := LogEntriesDropperFormatter{OriginalFormatter: entry.Logger.Formatter} - entry.Logger.Formatter = &formatter - return nil -} - -// LogEntriesDropperFormatter - custom formatter which will ignore log entries which has lower level than preconfigured in logger -type LogEntriesDropperFormatter struct { - OriginalFormatter logrus.Formatter -} - -// Format - custom entry formatting function which will drop entries with lower level than set in logger -func (formatter *LogEntriesDropperFormatter) Format(entry *logrus.Entry) ([]byte, error) { - if entry.Logger.Level >= entry.Level { - return formatter.OriginalFormatter.Format(entry) - } - return []byte(""), nil -} - -// ListStackDependentModules - build a map with each module and its dependent modules -func ListStackDependentModules(stack *Stack) map[string][]string { - // build map of dependent modules - // module path -> list of dependent modules - var dependentModules = make(map[string][]string) - - // build initial mapping of dependent modules - for _, module := range stack.Modules { - - if len(module.Dependencies) != 0 { - for _, dep := range module.Dependencies { - dependentModules[dep.Path] = util.RemoveDuplicatesFromList(append(dependentModules[dep.Path], module.Path)) - } - } - } - - // Floyd–Warshall inspired approach to find dependent modules - // merge map slices by key until no more updates are possible - - // Example: - // Initial setup: - // dependentModules["module1"] = ["module2", "module3"] - // dependentModules["module2"] = ["module3"] - // dependentModules["module3"] = ["module4"] - // dependentModules["module4"] = ["module5"] - - // After first iteration: (module1 += module4, module2 += module4, module3 += module5) - // dependentModules["module1"] = ["module2", "module3", "module4"] - // dependentModules["module2"] = ["module3", "module4"] - // dependentModules["module3"] = ["module4", "module5"] - // dependentModules["module4"] = ["module5"] - - // After second iteration: (module1 += module5, module2 += module5) - // dependentModules["module1"] = ["module2", "module3", "module4", "module5"] - // dependentModules["module2"] = ["module3", "module4", "module5"] - // dependentModules["module3"] = ["module4", "module5"] - // dependentModules["module4"] = ["module5"] - - // Done, no more updates and in map we have all dependent modules for each module. - - for { - noUpdates := true - for module, dependents := range dependentModules { - for _, dependent := range dependents { - initialSize := len(dependentModules[module]) - // merge without duplicates - list := util.RemoveDuplicatesFromList(append(dependentModules[module], dependentModules[dependent]...)) - list = util.RemoveElementFromList(list, module) - dependentModules[module] = list - if initialSize != len(dependentModules[module]) { - noUpdates = false - } - } - } - if noUpdates { - break - } - } - return dependentModules -} diff --git a/configstack/module_test.go b/configstack/module_test.go index e884693b4..415838b51 100644 --- a/configstack/module_test.go +++ b/configstack/module_test.go @@ -3,1031 +3,1230 @@ package configstack import ( "bytes" "context" - "os" - "path/filepath" - "reflect" + "fmt" "strings" "testing" - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/require" - "github.com/gruntwork-io/go-commons/errors" - "github.com/gruntwork-io/terragrunt/codegen" "github.com/gruntwork-io/terragrunt/config" "github.com/gruntwork-io/terragrunt/options" "github.com/stretchr/testify/assert" ) -var mockHowThesePathsWereFound = "mock-values-for-test" +func TestGraph(t *testing.T) { + a := &TerraformModule{Path: "a"} + b := &TerraformModule{Path: "b"} + c := &TerraformModule{Path: "c"} + d := &TerraformModule{Path: "d"} + e := &TerraformModule{Path: "e", Dependencies: []*TerraformModule{a}} + f := &TerraformModule{Path: "f", Dependencies: []*TerraformModule{a, b}} + g := &TerraformModule{Path: "g", Dependencies: []*TerraformModule{e}} + h := &TerraformModule{Path: "h", Dependencies: []*TerraformModule{g, f, c}} + + modules := TerraformModules{a, b, c, d, e, f, g, h} + + var stdout bytes.Buffer + terragruntOptions, _ := options.NewTerragruntOptionsForTest("/terragrunt.hcl") + modules.WriteDot(&stdout, terragruntOptions) + expected := strings.TrimSpace(` +digraph { + "a" ; + "b" ; + "c" ; + "d" ; + "e" ; + "e" -> "a"; + "f" ; + "f" -> "a"; + "f" -> "b"; + "g" ; + "g" -> "e"; + "h" ; + "h" -> "g"; + "h" -> "f"; + "h" -> "c"; +} +`) + assert.True(t, strings.Contains(stdout.String(), expected)) +} -func TestResolveTerraformModulesNoPaths(t *testing.T) { +func TestGraphTrimPrefix(t *testing.T) { + a := &TerraformModule{Path: "/config/a"} + b := &TerraformModule{Path: "/config/b"} + c := &TerraformModule{Path: "/config/c"} + d := &TerraformModule{Path: "/config/d"} + e := &TerraformModule{Path: "/config/alpha/beta/gamma/e", Dependencies: []*TerraformModule{a}} + f := &TerraformModule{Path: "/config/alpha/beta/gamma/f", Dependencies: []*TerraformModule{a, b}} + g := &TerraformModule{Path: "/config/alpha/g", Dependencies: []*TerraformModule{e}} + h := &TerraformModule{Path: "/config/alpha/beta/h", Dependencies: []*TerraformModule{g, f, c}} + + modules := TerraformModules{a, b, c, d, e, f, g, h} + + var stdout bytes.Buffer + terragruntOptions, _ := options.NewTerragruntOptionsWithConfigPath("/config/terragrunt.hcl") + modules.WriteDot(&stdout, terragruntOptions) + expected := strings.TrimSpace(` +digraph { + "a" ; + "b" ; + "c" ; + "d" ; + "alpha/beta/gamma/e" ; + "alpha/beta/gamma/e" -> "a"; + "alpha/beta/gamma/f" ; + "alpha/beta/gamma/f" -> "a"; + "alpha/beta/gamma/f" -> "b"; + "alpha/g" ; + "alpha/g" -> "alpha/beta/gamma/e"; + "alpha/beta/h" ; + "alpha/beta/h" -> "alpha/g"; + "alpha/beta/h" -> "alpha/beta/gamma/f"; + "alpha/beta/h" -> "c"; +} +`) + assert.True(t, strings.Contains(stdout.String(), expected)) +} + +func TestGraphFlagExcluded(t *testing.T) { + a := &TerraformModule{Path: "a", FlagExcluded: true} + b := &TerraformModule{Path: "b"} + c := &TerraformModule{Path: "c"} + d := &TerraformModule{Path: "d"} + e := &TerraformModule{Path: "e", Dependencies: []*TerraformModule{a}} + f := &TerraformModule{Path: "f", FlagExcluded: true, Dependencies: []*TerraformModule{a, b}} + g := &TerraformModule{Path: "g", Dependencies: []*TerraformModule{e}} + h := &TerraformModule{Path: "h", Dependencies: []*TerraformModule{g, f, c}} + + modules := TerraformModules{a, b, c, d, e, f, g, h} + + var stdout bytes.Buffer + terragruntOptions, _ := options.NewTerragruntOptionsForTest("/terragrunt.hcl") + modules.WriteDot(&stdout, terragruntOptions) + expected := strings.TrimSpace(` +digraph { + "a" [color=red]; + "b" ; + "c" ; + "d" ; + "e" ; + "e" -> "a"; + "f" [color=red]; + "f" -> "a"; + "f" -> "b"; + "g" ; + "g" -> "e"; + "h" ; + "h" -> "g"; + "h" -> "f"; + "h" -> "c"; +} +`) + assert.True(t, strings.Contains(stdout.String(), expected)) +} + +func TestCheckForCycles(t *testing.T) { t.Parallel() - configPaths := []string{} - expected := []*TerraformModule{} + //////////////////////////////////// + // These modules have no dependencies + //////////////////////////////////// + a := &TerraformModule{Path: "a"} + b := &TerraformModule{Path: "b"} + c := &TerraformModule{Path: "c"} + d := &TerraformModule{Path: "d"} + + //////////////////////////////////// + // These modules have dependencies, but no cycles + //////////////////////////////////// + + // e -> a + e := &TerraformModule{Path: "e", Dependencies: []*TerraformModule{a}} + + // f -> a, b + f := &TerraformModule{Path: "f", Dependencies: []*TerraformModule{a, b}} + + // g -> e -> a + g := &TerraformModule{Path: "g", Dependencies: []*TerraformModule{e}} + + // h -> g -> e -> a + // | / + // --> f -> b + // | + // --> c + h := &TerraformModule{Path: "h", Dependencies: []*TerraformModule{g, f, c}} + + //////////////////////////////////// + // These modules have dependencies and cycles + //////////////////////////////////// + + // i -> i + i := &TerraformModule{Path: "i", Dependencies: []*TerraformModule{}} + i.Dependencies = append(i.Dependencies, i) + + // j -> k -> j + j := &TerraformModule{Path: "j", Dependencies: []*TerraformModule{}} + k := &TerraformModule{Path: "k", Dependencies: []*TerraformModule{j}} + j.Dependencies = append(j.Dependencies, k) + + // l -> m -> n -> o -> l + l := &TerraformModule{Path: "l", Dependencies: []*TerraformModule{}} + o := &TerraformModule{Path: "o", Dependencies: []*TerraformModule{l}} + n := &TerraformModule{Path: "n", Dependencies: []*TerraformModule{o}} + m := &TerraformModule{Path: "m", Dependencies: []*TerraformModule{n}} + l.Dependencies = append(l.Dependencies, m) + + testCases := []struct { + modules TerraformModules + expected DependencyCycleError + }{ + {[]*TerraformModule{}, nil}, + {[]*TerraformModule{a}, nil}, + {[]*TerraformModule{a, b, c, d}, nil}, + {[]*TerraformModule{a, e}, nil}, + {[]*TerraformModule{a, b, f}, nil}, + {[]*TerraformModule{a, e, g}, nil}, + {[]*TerraformModule{a, b, c, e, f, g, h}, nil}, + {[]*TerraformModule{i}, DependencyCycleError([]string{"i", "i"})}, + {[]*TerraformModule{j, k}, DependencyCycleError([]string{"j", "k", "j"})}, + {[]*TerraformModule{l, o, n, m}, DependencyCycleError([]string{"l", "m", "n", "o", "l"})}, + {[]*TerraformModule{a, l, b, o, n, f, m, h}, DependencyCycleError([]string{"l", "m", "n", "o", "l"})}, + } - actualModules, actualErr := ResolveTerraformModules(context.Background(), configPaths, mockOptions, nil, mockHowThesePathsWereFound) - assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) - assertModuleListsEqual(t, expected, actualModules) + for _, testCase := range testCases { + actual := testCase.modules.CheckForCycles() + if testCase.expected == nil { + assert.Nil(t, actual) + } else if assert.NotNil(t, actual, "For modules %v", testCase.modules) { + actualErr := errors.Unwrap(actual).(DependencyCycleError) + assert.Equal(t, []string(testCase.expected), []string(actualErr), "For modules %v", testCase.modules) + } + } } -func TestResolveTerraformModulesOneModuleNoDependencies(t *testing.T) { +func TestRunModulesNoModules(t *testing.T) { t.Parallel() + opts, err := options.NewTerragruntOptionsForTest("") + assert.NoError(t, err) + + modules := TerraformModules{} + err = modules.RunModules(context.Background(), opts, options.DefaultParallelism) + assert.Nil(t, err, "Unexpected error: %v", err) +} + +func TestRunModulesOneModuleSuccess(t *testing.T) { + t.Parallel() + + aRan := false moduleA := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-a"), - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{ - Terraform: &config.TerraformConfig{Source: ptr("test")}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-a/"+config.DefaultTerragruntConfigPath)), - } - - configPaths := []string{"../test/fixture-modules/module-a/" + config.DefaultTerragruntConfigPath} - expected := []*TerraformModule{moduleA} - - actualModules, actualErr := ResolveTerraformModules(context.Background(), configPaths, mockOptions, nil, mockHowThesePathsWereFound) - assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) - assertModuleListsEqual(t, expected, actualModules) + Path: "a", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), + } + + opts, err := options.NewTerragruntOptionsForTest("") + assert.NoError(t, err) + + modules := TerraformModules{moduleA} + err = modules.RunModules(context.Background(), opts, options.DefaultParallelism) + assert.Nil(t, err, "Unexpected error: %v", err) + assert.True(t, aRan) } -func TestResolveTerraformModulesOneJsonModuleNoDependencies(t *testing.T) { +func TestRunModulesOneModuleAssumeAlreadyRan(t *testing.T) { t.Parallel() + aRan := false moduleA := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/json-module-a"), - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{ - Terraform: &config.TerraformConfig{Source: ptr("test")}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/json-module-a/"+config.DefaultTerragruntJsonConfigPath)), - } - - configPaths := []string{"../test/fixture-modules/json-module-a/" + config.DefaultTerragruntJsonConfigPath} - expected := []*TerraformModule{moduleA} - - actualModules, actualErr := ResolveTerraformModules(context.Background(), configPaths, mockOptions, nil, mockHowThesePathsWereFound) - assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) - assertModuleListsEqual(t, expected, actualModules) + Path: "a", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), + AssumeAlreadyApplied: true, + } + + opts, err := options.NewTerragruntOptionsForTest("") + assert.NoError(t, err) + + modules := TerraformModules{moduleA} + err = modules.RunModules(context.Background(), opts, options.DefaultParallelism) + assert.Nil(t, err, "Unexpected error: %v", err) + assert.False(t, aRan) } -func TestResolveTerraformModulesOneModuleWithIncludesNoDependencies(t *testing.T) { +func TestRunModulesReverseOrderOneModuleSuccess(t *testing.T) { t.Parallel() - moduleB := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-b/module-b-child"), - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{ - Terraform: &config.TerraformConfig{Source: ptr("...")}, - IsPartial: true, - ProcessedIncludes: map[string]config.IncludeConfig{ - "": {Path: canonical(t, "../test/fixture-modules/module-b/terragrunt.hcl")}, - }, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-b/module-b-child/"+config.DefaultTerragruntConfigPath)), - } - - configPaths := []string{"../test/fixture-modules/module-b/module-b-child/" + config.DefaultTerragruntConfigPath} - expected := []*TerraformModule{moduleB} - - actualModules, actualErr := ResolveTerraformModules(context.Background(), configPaths, mockOptions, nil, mockHowThesePathsWereFound) - assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) - assertModuleListsEqual(t, expected, actualModules) + aRan := false + moduleA := &TerraformModule{ + Path: "a", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), + } + + opts, err := options.NewTerragruntOptionsForTest("") + assert.NoError(t, err) + + modules := TerraformModules{moduleA} + err = modules.RunModulesReverseOrder(context.Background(), opts, options.DefaultParallelism) + assert.Nil(t, err, "Unexpected error: %v", err) + assert.True(t, aRan) } -func TestResolveTerraformModulesReadConfigFromParentConfig(t *testing.T) { +func TestRunModulesIgnoreOrderOneModuleSuccess(t *testing.T) { t.Parallel() - childDir := "../test/fixture-modules/module-m/module-m-child" - childConfigPath := filepath.Join(childDir, config.DefaultTerragruntConfigPath) - - parentDir := "../test/fixture-modules/module-m" - parentCofnigPath := filepath.Join(parentDir, config.DefaultTerragruntConfigPath) - - localsConfigPaths := map[string]string{ - "env_vars": "../test/fixture-modules/module-m/env.hcl", - "tier_vars": "../test/fixture-modules/module-m/module-m-child/tier.hcl", - } - - localsConfigs := make(map[string]interface{}) - - for name, configPath := range localsConfigPaths { - opts, err := options.NewTerragruntOptionsWithConfigPath(configPath) - assert.NoError(t, err) - - ctx := config.NewParsingContext(context.Background(), opts) - cfg, err := config.PartialParseConfigFile(ctx, configPath, nil) - assert.NoError(t, err) - - localsConfigs[name] = map[string]interface{}{ - "dependencies": interface{}(nil), - "download_dir": "", - "generate": map[string]interface{}{}, - "iam_assume_role_duration": interface{}(nil), - "iam_assume_role_session_name": "", - "iam_role": "", - "iam_web_identity_token": "", - "inputs": interface{}(nil), - "locals": cfg.Locals, - "retry_max_attempts": interface{}(nil), - "retry_sleep_interval_sec": interface{}(nil), - "retryable_errors": interface{}(nil), - "skip": false, - "terraform_binary": "", - "terraform_version_constraint": "", - "terragrunt_version_constraint": "", - } + aRan := false + moduleA := &TerraformModule{ + Path: "a", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), } - moduleM := &TerraformModule{ - Path: canonical(t, childDir), - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{ - Terraform: &config.TerraformConfig{Source: ptr("...")}, - IsPartial: true, - ProcessedIncludes: map[string]config.IncludeConfig{ - "": {Path: canonical(t, "../test/fixture-modules/module-m/terragrunt.hcl")}, - }, - Locals: localsConfigs, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - FieldsMetadata: map[string]map[string]interface{}{ - "locals-env_vars": { - "found_in_file": canonical(t, "../test/fixture-modules/module-m/terragrunt.hcl"), - }, - "locals-tier_vars": { - "found_in_file": canonical(t, "../test/fixture-modules/module-m/terragrunt.hcl"), - }, - }, - }, - TerragruntOptions: mockOptions.Clone(canonical(t, childConfigPath)), - } - - configPaths := []string{childConfigPath} - childTerragruntConfig := &config.TerragruntConfig{ - ProcessedIncludes: map[string]config.IncludeConfig{ - "": { - Path: parentCofnigPath, - }, - }, - } - expected := []*TerraformModule{moduleM} - - mockOptions, _ := options.NewTerragruntOptionsForTest("running_module_test") - mockOptions.OriginalTerragruntConfigPath = childConfigPath - - actualModules, actualErr := ResolveTerraformModules(context.Background(), configPaths, mockOptions, childTerragruntConfig, mockHowThesePathsWereFound) - assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) - assertModuleListsEqual(t, expected, actualModules) + opts, err := options.NewTerragruntOptionsForTest("") + assert.NoError(t, err) + + modules := TerraformModules{moduleA} + err = modules.RunModulesIgnoreOrder(context.Background(), opts, options.DefaultParallelism) + assert.Nil(t, err, "Unexpected error: %v", err) + assert.True(t, aRan) } -func TestResolveTerraformModulesOneJsonModuleWithIncludesNoDependencies(t *testing.T) { +func TestRunModulesOneModuleError(t *testing.T) { t.Parallel() - moduleB := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/json-module-b/module-b-child"), - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{ - Terraform: &config.TerraformConfig{Source: ptr("...")}, - IsPartial: true, - ProcessedIncludes: map[string]config.IncludeConfig{ - "": {Path: canonical(t, "../test/fixture-modules/json-module-b/terragrunt.hcl")}, - }, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/json-module-b/module-b-child/"+config.DefaultTerragruntJsonConfigPath)), - } - - configPaths := []string{"../test/fixture-modules/json-module-b/module-b-child/" + config.DefaultTerragruntJsonConfigPath} - expected := []*TerraformModule{moduleB} - - actualModules, actualErr := ResolveTerraformModules(context.Background(), configPaths, mockOptions, nil, mockHowThesePathsWereFound) - assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) - assertModuleListsEqual(t, expected, actualModules) + aRan := false + expectedErrA := fmt.Errorf("Expected error for module a") + moduleA := &TerraformModule{ + Path: "a", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", expectedErrA, &aRan), + } + + opts, err := options.NewTerragruntOptionsForTest("") + assert.NoError(t, err) + + modules := TerraformModules{moduleA} + err = modules.RunModules(context.Background(), opts, options.DefaultParallelism) + assertMultiErrorContains(t, err, expectedErrA) + assert.True(t, aRan) } -func TestResolveTerraformModulesOneHclModuleWithIncludesNoDependencies(t *testing.T) { +func TestRunModulesReverseOrderOneModuleError(t *testing.T) { t.Parallel() - moduleB := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/hcl-module-b/module-b-child"), - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{ - Terraform: &config.TerraformConfig{Source: ptr("...")}, - IsPartial: true, - ProcessedIncludes: map[string]config.IncludeConfig{ - "": {Path: canonical(t, "../test/fixture-modules/hcl-module-b/terragrunt.hcl.json")}, - }, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/hcl-module-b/module-b-child/"+config.DefaultTerragruntConfigPath)), - } - - configPaths := []string{"../test/fixture-modules/hcl-module-b/module-b-child/" + config.DefaultTerragruntConfigPath} - expected := []*TerraformModule{moduleB} - - actualModules, actualErr := ResolveTerraformModules(context.Background(), configPaths, mockOptions, nil, mockHowThesePathsWereFound) - assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) - assertModuleListsEqual(t, expected, actualModules) + aRan := false + expectedErrA := fmt.Errorf("Expected error for module a") + moduleA := &TerraformModule{ + Path: "a", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", expectedErrA, &aRan), + } + + opts, err := options.NewTerragruntOptionsForTest("") + assert.NoError(t, err) + + modules := TerraformModules{moduleA} + err = modules.RunModulesReverseOrder(context.Background(), opts, options.DefaultParallelism) + assertMultiErrorContains(t, err, expectedErrA) + assert.True(t, aRan) } -func TestResolveTerraformModulesTwoModulesWithDependencies(t *testing.T) { +func TestRunModulesIgnoreOrderOneModuleError(t *testing.T) { t.Parallel() + aRan := false + expectedErrA := fmt.Errorf("Expected error for module a") moduleA := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-a"), - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{ - Terraform: &config.TerraformConfig{Source: ptr("test")}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-a/"+config.DefaultTerragruntConfigPath)), + Path: "a", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", expectedErrA, &aRan), } - moduleC := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-c"), - Dependencies: []*TerraformModule{moduleA}, - Config: config.TerragruntConfig{ - Dependencies: &config.ModuleDependencies{Paths: []string{"../module-a"}}, - Terraform: &config.TerraformConfig{Source: ptr("temp")}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-c/"+config.DefaultTerragruntConfigPath)), - } - - configPaths := []string{"../test/fixture-modules/module-a/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-c/" + config.DefaultTerragruntConfigPath} - expected := []*TerraformModule{moduleA, moduleC} - - actualModules, actualErr := ResolveTerraformModules(context.Background(), configPaths, mockOptions, nil, mockHowThesePathsWereFound) - assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) - assertModuleListsEqual(t, expected, actualModules) + opts, err := options.NewTerragruntOptionsForTest("") + assert.NoError(t, err) + + modules := TerraformModules{moduleA} + err = modules.RunModulesIgnoreOrder(context.Background(), opts, options.DefaultParallelism) + assertMultiErrorContains(t, err, expectedErrA) + assert.True(t, aRan) } -func TestResolveTerraformModulesJsonModulesWithHclDependencies(t *testing.T) { +func TestRunModulesMultipleModulesNoDependenciesSuccess(t *testing.T) { t.Parallel() + aRan := false moduleA := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-a"), - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{ - Terraform: &config.TerraformConfig{Source: ptr("test")}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-a/"+config.DefaultTerragruntConfigPath)), + Path: "a", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), } + bRan := false + moduleB := &TerraformModule{ + Path: "b", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", nil, &bRan), + } + + cRan := false moduleC := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/json-module-c"), - Dependencies: []*TerraformModule{moduleA}, - Config: config.TerragruntConfig{ - Dependencies: &config.ModuleDependencies{Paths: []string{"../module-a"}}, - Terraform: &config.TerraformConfig{Source: ptr("temp")}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/json-module-c/"+config.DefaultTerragruntJsonConfigPath)), - } - - configPaths := []string{"../test/fixture-modules/module-a/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/json-module-c/" + config.DefaultTerragruntJsonConfigPath} - expected := []*TerraformModule{moduleA, moduleC} - - actualModules, actualErr := ResolveTerraformModules(context.Background(), configPaths, mockOptions, nil, mockHowThesePathsWereFound) - assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) - assertModuleListsEqual(t, expected, actualModules) + Path: "c", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), + } + + opts, err := options.NewTerragruntOptionsForTest("") + assert.NoError(t, err) + + modules := TerraformModules{moduleA, moduleB, moduleC} + err = modules.RunModules(context.Background(), opts, options.DefaultParallelism) + assert.Nil(t, err, "Unexpected error: %v", err) + + assert.True(t, aRan) + assert.True(t, bRan) + assert.True(t, cRan) } -func TestResolveTerraformModulesHclModulesWithJsonDependencies(t *testing.T) { +func TestRunModulesMultipleModulesNoDependenciesSuccessNoParallelism(t *testing.T) { t.Parallel() + aRan := false moduleA := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/json-module-a"), - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{ - Terraform: &config.TerraformConfig{Source: ptr("test")}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/json-module-a/"+config.DefaultTerragruntJsonConfigPath)), + Path: "a", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), + } + + bRan := false + moduleB := &TerraformModule{ + Path: "b", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", nil, &bRan), } + cRan := false moduleC := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/hcl-module-c"), - Dependencies: []*TerraformModule{moduleA}, - Config: config.TerragruntConfig{ - Dependencies: &config.ModuleDependencies{Paths: []string{"../json-module-a"}}, - Terraform: &config.TerraformConfig{Source: ptr("temp")}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/hcl-module-c/"+config.DefaultTerragruntConfigPath)), - } - - configPaths := []string{"../test/fixture-modules/json-module-a/" + config.DefaultTerragruntJsonConfigPath, "../test/fixture-modules/hcl-module-c/" + config.DefaultTerragruntConfigPath} - expected := []*TerraformModule{moduleA, moduleC} - - actualModules, actualErr := ResolveTerraformModules(context.Background(), configPaths, mockOptions, nil, mockHowThesePathsWereFound) - assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) - assertModuleListsEqual(t, expected, actualModules) + Path: "c", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), + } + + opts, err := options.NewTerragruntOptionsForTest("") + assert.NoError(t, err) + + modules := TerraformModules{moduleA, moduleB, moduleC} + err = modules.RunModules(context.Background(), opts, 1) + assert.Nil(t, err, "Unexpected error: %v", err) + + assert.True(t, aRan) + assert.True(t, bRan) + assert.True(t, cRan) } -func TestResolveTerraformModulesTwoModulesWithDependenciesExcludedDirsWithDependency(t *testing.T) { +func TestRunModulesReverseOrderMultipleModulesNoDependenciesSuccess(t *testing.T) { t.Parallel() - opts, _ := options.NewTerragruntOptionsForTest("running_module_test") - opts.ExcludeDirs = []string{canonical(t, "../test/fixture-modules/module-a")} - + aRan := false moduleA := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-a"), - Dependencies: []*TerraformModule{}, - TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-a/"+config.DefaultTerragruntConfigPath)), + Path: "a", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), } - moduleC := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-c"), - Dependencies: []*TerraformModule{moduleA}, - Config: config.TerragruntConfig{ - Dependencies: &config.ModuleDependencies{Paths: []string{"../module-a"}}, - Terraform: &config.TerraformConfig{Source: ptr("temp")}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-c/"+config.DefaultTerragruntConfigPath)), + bRan := false + moduleB := &TerraformModule{ + Path: "b", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", nil, &bRan), } - configPaths := []string{"../test/fixture-modules/module-a/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-c/" + config.DefaultTerragruntConfigPath} + cRan := false + moduleC := &TerraformModule{ + Path: "c", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), + } - actualModules, actualErr := ResolveTerraformModules(context.Background(), configPaths, opts, nil, mockHowThesePathsWereFound) + opts, err := options.NewTerragruntOptionsForTest("") + assert.NoError(t, err) - // construct the expected list - moduleA.FlagExcluded = true - expected := []*TerraformModule{moduleA, moduleC} + modules := TerraformModules{moduleA, moduleB, moduleC} + err = modules.RunModulesReverseOrder(context.Background(), opts, options.DefaultParallelism) + assert.Nil(t, err, "Unexpected error: %v", err) - assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) - assertModuleListsEqual(t, expected, actualModules) + assert.True(t, aRan) + assert.True(t, bRan) + assert.True(t, cRan) } -func TestResolveTerraformModulesTwoModulesWithDependenciesExcludedDirsWithDependencyAndConflictingNaming(t *testing.T) { +func TestRunModulesIgnoreOrderMultipleModulesNoDependenciesSuccess(t *testing.T) { t.Parallel() - opts, _ := options.NewTerragruntOptionsForTest("running_module_test") - opts.ExcludeDirs = []string{canonical(t, "../test/fixture-modules/module-a")} - + aRan := false moduleA := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-a"), - Dependencies: []*TerraformModule{}, - TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-a/"+config.DefaultTerragruntConfigPath)), + Path: "a", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), + } + + bRan := false + moduleB := &TerraformModule{ + Path: "b", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", nil, &bRan), } + cRan := false moduleC := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-c"), - Dependencies: []*TerraformModule{moduleA}, - Config: config.TerragruntConfig{ - Dependencies: &config.ModuleDependencies{Paths: []string{"../module-a"}}, - Terraform: &config.TerraformConfig{Source: ptr("temp")}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-c/"+config.DefaultTerragruntConfigPath)), - } - - moduleAbba := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-abba"), - Dependencies: []*TerraformModule{moduleA}, - Config: config.TerragruntConfig{ - Dependencies: &config.ModuleDependencies{Paths: []string{"../module-a"}}, - Terraform: &config.TerraformConfig{Source: ptr("temp")}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-abba/"+config.DefaultTerragruntConfigPath)), - } - - configPaths := []string{"../test/fixture-modules/module-a/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-c/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-abba/" + config.DefaultTerragruntConfigPath} - - actualModules, actualErr := ResolveTerraformModules(context.Background(), configPaths, opts, nil, mockHowThesePathsWereFound) - - // construct the expected list - moduleA.FlagExcluded = true - expected := []*TerraformModule{moduleA, moduleC, moduleAbba} - - assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) - assertModuleListsEqual(t, expected, actualModules) + Path: "c", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), + } + + opts, err := options.NewTerragruntOptionsForTest("") + assert.NoError(t, err) + + modules := TerraformModules{moduleA, moduleB, moduleC} + err = modules.RunModulesIgnoreOrder(context.Background(), opts, options.DefaultParallelism) + assert.Nil(t, err, "Unexpected error: %v", err) + + assert.True(t, aRan) + assert.True(t, bRan) + assert.True(t, cRan) } -func TestResolveTerraformModulesTwoModulesWithDependenciesExcludedDirsWithDependencyAndConflictingNamingAndGlob(t *testing.T) { +func TestRunModulesMultipleModulesNoDependenciesOneFailure(t *testing.T) { t.Parallel() - opts, _ := options.NewTerragruntOptionsForTest("running_module_test") - opts.ExcludeDirs = globCanonical(t, "../test/fixture-modules/module-a*") - + aRan := false moduleA := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-a"), - Dependencies: []*TerraformModule{}, - TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-a/"+config.DefaultTerragruntConfigPath)), + Path: "a", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), } - moduleC := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-c"), - Dependencies: []*TerraformModule{moduleA}, - Config: config.TerragruntConfig{ - Dependencies: &config.ModuleDependencies{Paths: []string{"../module-a"}}, - Terraform: &config.TerraformConfig{Source: ptr("temp")}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-c/"+config.DefaultTerragruntConfigPath)), + bRan := false + expectedErrB := fmt.Errorf("Expected error for module b") + moduleB := &TerraformModule{ + Path: "b", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", expectedErrB, &bRan), } - moduleAbba := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-abba"), - Dependencies: []*TerraformModule{}, - TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-abba/"+config.DefaultTerragruntConfigPath)), + cRan := false + moduleC := &TerraformModule{ + Path: "c", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), } - configPaths := []string{"../test/fixture-modules/module-a/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-c/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-abba/" + config.DefaultTerragruntConfigPath} + opts, optsErr := options.NewTerragruntOptionsForTest("") + assert.NoError(t, optsErr) - actualModules, actualErr := ResolveTerraformModules(context.Background(), configPaths, opts, nil, mockHowThesePathsWereFound) - // construct the expected list - moduleA.FlagExcluded = true - moduleAbba.FlagExcluded = true - expected := []*TerraformModule{moduleA, moduleC, moduleAbba} + modules := TerraformModules{moduleA, moduleB, moduleC} + err := modules.RunModules(context.Background(), opts, options.DefaultParallelism) + assertMultiErrorContains(t, err, expectedErrB) - assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) - assertModuleListsEqual(t, expected, actualModules) + assert.True(t, aRan) + assert.True(t, bRan) + assert.True(t, cRan) } -func TestResolveTerraformModulesTwoModulesWithDependenciesExcludedDirsWithNoDependency(t *testing.T) { +func TestRunModulesMultipleModulesNoDependenciesMultipleFailures(t *testing.T) { t.Parallel() - opts, _ := options.NewTerragruntOptionsForTest("running_module_test") - opts.ExcludeDirs = []string{canonical(t, "../test/fixture-modules/module-c")} - + aRan := false + expectedErrA := fmt.Errorf("Expected error for module a") moduleA := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-a"), - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{ - Terraform: &config.TerraformConfig{Source: ptr("test")}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-a/"+config.DefaultTerragruntConfigPath)), + Path: "a", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", expectedErrA, &aRan), } - moduleC := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-c"), - TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-c/"+config.DefaultTerragruntConfigPath)), + bRan := false + expectedErrB := fmt.Errorf("Expected error for module b") + moduleB := &TerraformModule{ + Path: "b", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", expectedErrB, &bRan), } - configPaths := []string{"../test/fixture-modules/module-a/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-c/" + config.DefaultTerragruntConfigPath} + cRan := false + expectedErrC := fmt.Errorf("Expected error for module c") + moduleC := &TerraformModule{ + Path: "c", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", expectedErrC, &cRan), + } - actualModules, actualErr := ResolveTerraformModules(context.Background(), configPaths, opts, nil, mockHowThesePathsWereFound) + opts, err := options.NewTerragruntOptionsForTest("") + assert.NoError(t, err) - // construct the expected list - moduleC.FlagExcluded = true - expected := []*TerraformModule{moduleA, moduleC} + modules := TerraformModules{moduleA, moduleB, moduleC} + err = modules.RunModules(context.Background(), opts, options.DefaultParallelism) + assertMultiErrorContains(t, err, expectedErrA, expectedErrB, expectedErrC) - assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) - assertModuleListsEqual(t, expected, actualModules) + assert.True(t, aRan) + assert.True(t, bRan) + assert.True(t, cRan) } -func TestResolveTerraformModulesTwoModulesWithDependenciesIncludedDirsWithDependency(t *testing.T) { +func TestRunModulesMultipleModulesWithDependenciesSuccess(t *testing.T) { t.Parallel() - opts, _ := options.NewTerragruntOptionsForTest("running_module_test") - opts.IncludeDirs = []string{canonical(t, "../test/fixture-modules/module-c")} - + aRan := false moduleA := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-a"), - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{ - Terraform: &config.TerraformConfig{Source: ptr("test")}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-a/"+config.DefaultTerragruntConfigPath)), + Path: "a", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), } - moduleC := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-c"), - Dependencies: []*TerraformModule{moduleA}, - Config: config.TerragruntConfig{ - Dependencies: &config.ModuleDependencies{Paths: []string{"../module-a"}}, - Terraform: &config.TerraformConfig{Source: ptr("temp")}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-c/"+config.DefaultTerragruntConfigPath)), + bRan := false + moduleB := &TerraformModule{ + Path: "b", + Dependencies: TerraformModules{moduleA}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", nil, &bRan), } - configPaths := []string{"../test/fixture-modules/module-a/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-c/" + config.DefaultTerragruntConfigPath} + cRan := false + moduleC := &TerraformModule{ + Path: "c", + Dependencies: TerraformModules{moduleB}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), + } - actualModules, actualErr := ResolveTerraformModules(context.Background(), configPaths, opts, nil, mockHowThesePathsWereFound) + opts, err := options.NewTerragruntOptionsForTest("") + assert.NoError(t, err) - // construct the expected list - moduleA.FlagExcluded = false - expected := []*TerraformModule{moduleA, moduleC} + modules := TerraformModules{moduleA, moduleB, moduleC} + err = modules.RunModules(context.Background(), opts, options.DefaultParallelism) + assert.Nil(t, err, "Unexpected error: %v", err) - assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) - assertModuleListsEqual(t, expected, actualModules) + assert.True(t, aRan) + assert.True(t, bRan) + assert.True(t, cRan) } -func TestResolveTerraformModulesTwoModulesWithDependenciesIncludedDirsWithNoDependency(t *testing.T) { +func TestRunModulesMultipleModulesWithDependenciesWithAssumeAlreadyRanSuccess(t *testing.T) { t.Parallel() - opts, _ := options.NewTerragruntOptionsForTest("running_module_test") - opts.IncludeDirs = []string{canonical(t, "../test/fixture-modules/module-a")} - + aRan := false moduleA := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-a"), - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{ - Terraform: &config.TerraformConfig{Source: ptr("test")}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-a/"+config.DefaultTerragruntConfigPath)), + Path: "a", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), + } + + bRan := false + moduleB := &TerraformModule{ + Path: "b", + Dependencies: TerraformModules{moduleA}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", nil, &bRan), } + cRan := false moduleC := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-c"), - Dependencies: []*TerraformModule{moduleA}, - Config: config.TerragruntConfig{ - Dependencies: &config.ModuleDependencies{Paths: []string{"../module-a"}}, - Terraform: &config.TerraformConfig{Source: ptr("temp")}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-c/"+config.DefaultTerragruntConfigPath)), + Path: "c", + Dependencies: TerraformModules{moduleB}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), + AssumeAlreadyApplied: true, } - configPaths := []string{"../test/fixture-modules/module-a/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-c/" + config.DefaultTerragruntConfigPath} + dRan := false + moduleD := &TerraformModule{ + Path: "d", + Dependencies: TerraformModules{moduleC}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "d", nil, &dRan), + } - actualModules, actualErr := ResolveTerraformModules(context.Background(), configPaths, opts, nil, mockHowThesePathsWereFound) + opts, err := options.NewTerragruntOptionsForTest("") + assert.NoError(t, err) - // construct the expected list - moduleC.FlagExcluded = true - expected := []*TerraformModule{moduleA, moduleC} + modules := TerraformModules{moduleA, moduleB, moduleC, moduleD} + err = modules.RunModules(context.Background(), opts, options.DefaultParallelism) + assert.Nil(t, err, "Unexpected error: %v", err) - assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) - assertModuleListsEqual(t, expected, actualModules) + assert.True(t, aRan) + assert.True(t, bRan) + assert.False(t, cRan) + assert.True(t, dRan) } -func TestResolveTerraformModulesTwoModulesWithDependenciesIncludedDirsWithDependencyExcludeModuleWithNoDependency(t *testing.T) { +func TestRunModulesReverseOrderMultipleModulesWithDependenciesSuccess(t *testing.T) { t.Parallel() - opts, _ := options.NewTerragruntOptionsForTest("running_module_test") - opts.IncludeDirs = []string{canonical(t, "../test/fixture-modules/module-c"), canonical(t, "../test/fixture-modules/module-f")} - opts.ExcludeDirs = []string{canonical(t, "../test/fixture-modules/module-f")} - + aRan := false moduleA := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-a"), - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{ - Terraform: &config.TerraformConfig{Source: ptr("test")}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-a/"+config.DefaultTerragruntConfigPath)), + Path: "a", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), } - moduleC := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-c"), - Dependencies: []*TerraformModule{moduleA}, - Config: config.TerragruntConfig{ - Dependencies: &config.ModuleDependencies{Paths: []string{"../module-a"}}, - Terraform: &config.TerraformConfig{Source: ptr("temp")}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-c/"+config.DefaultTerragruntConfigPath)), + bRan := false + moduleB := &TerraformModule{ + Path: "b", + Dependencies: TerraformModules{moduleA}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", nil, &bRan), } - moduleF := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-f"), - Dependencies: []*TerraformModule{}, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-f/"+config.DefaultTerragruntConfigPath)), - AssumeAlreadyApplied: false, + cRan := false + moduleC := &TerraformModule{ + Path: "c", + Dependencies: TerraformModules{moduleB}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), } - configPaths := []string{"../test/fixture-modules/module-a/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-c/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-f/" + config.DefaultTerragruntConfigPath} - - actualModules, actualErr := ResolveTerraformModules(context.Background(), configPaths, opts, nil, mockHowThesePathsWereFound) + opts, err := options.NewTerragruntOptionsForTest("") + assert.NoError(t, err) - // construct the expected list - moduleF.FlagExcluded = true - expected := []*TerraformModule{moduleA, moduleC, moduleF} + modules := TerraformModules{moduleA, moduleB, moduleC} + err = modules.RunModulesReverseOrder(context.Background(), opts, options.DefaultParallelism) + assert.Nil(t, err, "Unexpected error: %v", err) - assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) - assertModuleListsEqual(t, expected, actualModules) + assert.True(t, aRan) + assert.True(t, bRan) + assert.True(t, cRan) } -func TestResolveTerraformModulesMultipleModulesWithDependencies(t *testing.T) { +func TestRunModulesIgnoreOrderMultipleModulesWithDependenciesSuccess(t *testing.T) { t.Parallel() + aRan := false moduleA := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-a"), - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{ - Terraform: &config.TerraformConfig{Source: ptr("test")}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-a/"+config.DefaultTerragruntConfigPath)), + Path: "a", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), } + bRan := false moduleB := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-b/module-b-child"), - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{ - Terraform: &config.TerraformConfig{Source: ptr("...")}, - IsPartial: true, - ProcessedIncludes: map[string]config.IncludeConfig{ - "": {Path: canonical(t, "../test/fixture-modules/module-b/terragrunt.hcl")}, - }, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-b/module-b-child/"+config.DefaultTerragruntConfigPath)), + Path: "b", + Dependencies: TerraformModules{moduleA}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", nil, &bRan), } + cRan := false moduleC := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-c"), - Dependencies: []*TerraformModule{moduleA}, - Config: config.TerragruntConfig{ - Dependencies: &config.ModuleDependencies{Paths: []string{"../module-a"}}, - Terraform: &config.TerraformConfig{Source: ptr("temp")}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-c/"+config.DefaultTerragruntConfigPath)), + Path: "c", + Dependencies: TerraformModules{moduleB}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), } - moduleD := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-d"), - Dependencies: []*TerraformModule{moduleA, moduleB, moduleC}, - Config: config.TerragruntConfig{ - Dependencies: &config.ModuleDependencies{Paths: []string{"../module-a", "../module-b/module-b-child", "../module-c"}}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-d/"+config.DefaultTerragruntConfigPath)), - } - - configPaths := []string{"../test/fixture-modules/module-a/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-b/module-b-child/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-c/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-d/" + config.DefaultTerragruntConfigPath} - expected := []*TerraformModule{moduleA, moduleB, moduleC, moduleD} - - actualModules, actualErr := ResolveTerraformModules(context.Background(), configPaths, mockOptions, nil, mockHowThesePathsWereFound) - assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) - assertModuleListsEqual(t, expected, actualModules) + opts, err := options.NewTerragruntOptionsForTest("") + assert.NoError(t, err) + + modules := TerraformModules{moduleA, moduleB, moduleC} + err = modules.RunModulesIgnoreOrder(context.Background(), opts, options.DefaultParallelism) + assert.Nil(t, err, "Unexpected error: %v", err) + + assert.True(t, aRan) + assert.True(t, bRan) + assert.True(t, cRan) } -func TestResolveTerraformModulesMultipleModulesWithMixedDependencies(t *testing.T) { +func TestRunModulesMultipleModulesWithDependenciesOneFailure(t *testing.T) { t.Parallel() + aRan := false moduleA := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-a"), - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{ - Terraform: &config.TerraformConfig{Source: ptr("test")}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-a/"+config.DefaultTerragruntConfigPath)), + Path: "a", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), } + bRan := false + expectedErrB := fmt.Errorf("Expected error for module b") moduleB := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/json-module-b/module-b-child"), - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{ - Terraform: &config.TerraformConfig{Source: ptr("...")}, - IsPartial: true, - ProcessedIncludes: map[string]config.IncludeConfig{ - "": {Path: canonical(t, "../test/fixture-modules/json-module-b/terragrunt.hcl")}, - }, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/json-module-b/module-b-child/"+config.DefaultTerragruntJsonConfigPath)), + Path: "b", + Dependencies: TerraformModules{moduleA}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", expectedErrB, &bRan), } + cRan := false moduleC := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-c"), - Dependencies: []*TerraformModule{moduleA}, - Config: config.TerragruntConfig{ - Dependencies: &config.ModuleDependencies{Paths: []string{"../module-a"}}, - Terraform: &config.TerraformConfig{Source: ptr("temp")}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-c/"+config.DefaultTerragruntConfigPath)), + Path: "c", + Dependencies: TerraformModules{moduleB}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), } - moduleD := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/json-module-d"), - Dependencies: []*TerraformModule{moduleA, moduleB, moduleC}, - Config: config.TerragruntConfig{ - Dependencies: &config.ModuleDependencies{Paths: []string{"../module-a", "../json-module-b/module-b-child", "../module-c"}}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/json-module-d/"+config.DefaultTerragruntJsonConfigPath)), - } - - configPaths := []string{"../test/fixture-modules/module-a/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/json-module-b/module-b-child/" + config.DefaultTerragruntJsonConfigPath, "../test/fixture-modules/module-c/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/json-module-d/" + config.DefaultTerragruntJsonConfigPath} - expected := []*TerraformModule{moduleA, moduleB, moduleC, moduleD} - - actualModules, actualErr := ResolveTerraformModules(context.Background(), configPaths, mockOptions, nil, mockHowThesePathsWereFound) - assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) - assertModuleListsEqual(t, expected, actualModules) + expectedErrC := ProcessingModuleDependencyError{moduleC, moduleB, expectedErrB} + + opts, err := options.NewTerragruntOptionsForTest("") + assert.NoError(t, err) + + modules := TerraformModules{moduleA, moduleB, moduleC} + err = modules.RunModules(context.Background(), opts, options.DefaultParallelism) + assertMultiErrorContains(t, err, expectedErrB, expectedErrC) + + assert.True(t, aRan) + assert.True(t, bRan) + assert.False(t, cRan) } -func TestResolveTerraformModulesMultipleModulesWithDependenciesWithIncludes(t *testing.T) { +func TestRunModulesMultipleModulesWithDependenciesOneFailureIgnoreDependencyErrors(t *testing.T) { t.Parallel() + aRan := false + terragruntOptionsA := optionsWithMockTerragruntCommand(t, "a", nil, &aRan) + terragruntOptionsA.IgnoreDependencyErrors = true moduleA := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-a"), - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{ - Terraform: &config.TerraformConfig{Source: ptr("test")}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-a/"+config.DefaultTerragruntConfigPath)), + Path: "a", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: terragruntOptionsA, } + bRan := false + expectedErrB := fmt.Errorf("Expected error for module b") + terragruntOptionsB := optionsWithMockTerragruntCommand(t, "b", expectedErrB, &bRan) + terragruntOptionsB.IgnoreDependencyErrors = true moduleB := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-b/module-b-child"), - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{ - Terraform: &config.TerraformConfig{Source: ptr("...")}, - IsPartial: true, - ProcessedIncludes: map[string]config.IncludeConfig{ - "": {Path: canonical(t, "../test/fixture-modules/module-b/terragrunt.hcl")}, - }, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-b/module-b-child/"+config.DefaultTerragruntConfigPath)), + Path: "b", + Dependencies: TerraformModules{moduleA}, + Config: config.TerragruntConfig{}, + TerragruntOptions: terragruntOptionsB, } - moduleE := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-e/module-e-child"), - Dependencies: []*TerraformModule{moduleA, moduleB}, - Config: config.TerragruntConfig{ - Dependencies: &config.ModuleDependencies{Paths: []string{"../../module-a", "../../module-b/module-b-child"}}, - Terraform: &config.TerraformConfig{Source: ptr("test")}, - IsPartial: true, - ProcessedIncludes: map[string]config.IncludeConfig{ - "": {Path: canonical(t, "../test/fixture-modules/module-e/terragrunt.hcl")}, - }, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-e/module-e-child/"+config.DefaultTerragruntConfigPath)), - } - - configPaths := []string{"../test/fixture-modules/module-a/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-b/module-b-child/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-e/module-e-child/" + config.DefaultTerragruntConfigPath} - expected := []*TerraformModule{moduleA, moduleB, moduleE} - - actualModules, actualErr := ResolveTerraformModules(context.Background(), configPaths, mockOptions, nil, mockHowThesePathsWereFound) - assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) - assertModuleListsEqual(t, expected, actualModules) + cRan := false + terragruntOptionsC := optionsWithMockTerragruntCommand(t, "c", nil, &cRan) + terragruntOptionsC.IgnoreDependencyErrors = true + moduleC := &TerraformModule{ + Path: "c", + Dependencies: TerraformModules{moduleB}, + Config: config.TerragruntConfig{}, + TerragruntOptions: terragruntOptionsC, + } + + opts, err := options.NewTerragruntOptionsForTest("") + assert.NoError(t, err) + + modules := TerraformModules{moduleA, moduleB, moduleC} + err = modules.RunModules(context.Background(), opts, options.DefaultParallelism) + assertMultiErrorContains(t, err, expectedErrB) + + assert.True(t, aRan) + assert.True(t, bRan) + assert.True(t, cRan) } -func TestResolveTerraformModulesMultipleModulesWithExternalDependencies(t *testing.T) { +func TestRunModulesReverseOrderMultipleModulesWithDependenciesOneFailure(t *testing.T) { t.Parallel() - moduleF := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-f"), - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{ - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-f/"+config.DefaultTerragruntConfigPath)), - AssumeAlreadyApplied: true, + aRan := false + moduleA := &TerraformModule{ + Path: "a", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), } - moduleG := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-g"), - Dependencies: []*TerraformModule{moduleF}, - Config: config.TerragruntConfig{ - Dependencies: &config.ModuleDependencies{Paths: []string{"../module-f"}}, - Terraform: &config.TerraformConfig{Source: ptr("test")}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-g/"+config.DefaultTerragruntConfigPath)), - } - - configPaths := []string{"../test/fixture-modules/module-g/" + config.DefaultTerragruntConfigPath} - expected := []*TerraformModule{moduleF, moduleG} - - actualModules, actualErr := ResolveTerraformModules(context.Background(), configPaths, mockOptions, nil, mockHowThesePathsWereFound) - assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) - assertModuleListsEqual(t, expected, actualModules) + bRan := false + expectedErrB := fmt.Errorf("Expected error for module b") + moduleB := &TerraformModule{ + Path: "b", + Dependencies: TerraformModules{moduleA}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", expectedErrB, &bRan), + } + + cRan := false + moduleC := &TerraformModule{ + Path: "c", + Dependencies: TerraformModules{moduleB}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), + } + + expectedErrA := ProcessingModuleDependencyError{moduleA, moduleB, expectedErrB} + + opts, err := options.NewTerragruntOptionsForTest("") + assert.NoError(t, err) + + modules := TerraformModules{moduleA, moduleB, moduleC} + err = modules.RunModulesReverseOrder(context.Background(), opts, options.DefaultParallelism) + assertMultiErrorContains(t, err, expectedErrB, expectedErrA) + + assert.False(t, aRan) + assert.True(t, bRan) + assert.True(t, cRan) } -func TestResolveTerraformModulesMultipleModulesWithNestedExternalDependencies(t *testing.T) { +func TestRunModulesIgnoreOrderMultipleModulesWithDependenciesOneFailure(t *testing.T) { t.Parallel() - moduleH := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-h"), - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{ - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-h/"+config.DefaultTerragruntConfigPath)), - AssumeAlreadyApplied: true, + aRan := false + moduleA := &TerraformModule{ + Path: "a", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), } - moduleI := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-i"), - Dependencies: []*TerraformModule{moduleH}, - Config: config.TerragruntConfig{ - Dependencies: &config.ModuleDependencies{Paths: []string{"../module-h"}}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-i/"+config.DefaultTerragruntConfigPath)), - AssumeAlreadyApplied: true, + bRan := false + expectedErrB := fmt.Errorf("Expected error for module b") + moduleB := &TerraformModule{ + Path: "b", + Dependencies: TerraformModules{moduleA}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", expectedErrB, &bRan), + } + + cRan := false + moduleC := &TerraformModule{ + Path: "c", + Dependencies: TerraformModules{moduleB}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), } - moduleJ := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-j"), - Dependencies: []*TerraformModule{moduleI}, - Config: config.TerragruntConfig{ - Dependencies: &config.ModuleDependencies{Paths: []string{"../module-i"}}, - Terraform: &config.TerraformConfig{Source: ptr("temp")}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-j/"+config.DefaultTerragruntConfigPath)), - } - - moduleK := &TerraformModule{ - Path: canonical(t, "../test/fixture-modules/module-k"), - Dependencies: []*TerraformModule{moduleH}, - Config: config.TerragruntConfig{ - Dependencies: &config.ModuleDependencies{Paths: []string{"../module-h"}}, - Terraform: &config.TerraformConfig{Source: ptr("fire")}, - IsPartial: true, - GenerateConfigs: make(map[string]codegen.GenerateConfig), - }, - TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-k/"+config.DefaultTerragruntConfigPath)), - } - - configPaths := []string{"../test/fixture-modules/module-j/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-k/" + config.DefaultTerragruntConfigPath} - expected := []*TerraformModule{moduleH, moduleI, moduleJ, moduleK} - - actualModules, actualErr := ResolveTerraformModules(context.Background(), configPaths, mockOptions, nil, mockHowThesePathsWereFound) - require.NoError(t, actualErr) - assertModuleListsEqual(t, expected, actualModules) + opts, err := options.NewTerragruntOptionsForTest("") + assert.NoError(t, err) + + modules := TerraformModules{moduleA, moduleB, moduleC} + err = modules.RunModulesIgnoreOrder(context.Background(), opts, options.DefaultParallelism) + assertMultiErrorContains(t, err, expectedErrB) + + assert.True(t, aRan) + assert.True(t, bRan) + assert.True(t, cRan) } -func TestResolveTerraformModulesInvalidPaths(t *testing.T) { +func TestRunModulesMultipleModulesWithDependenciesMultipleFailures(t *testing.T) { t.Parallel() - configPaths := []string{"../test/fixture-modules/module-missing-dependency/" + config.DefaultTerragruntConfigPath} + aRan := false + expectedErrA := fmt.Errorf("Expected error for module a") + moduleA := &TerraformModule{ + Path: "a", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", expectedErrA, &aRan), + } - _, actualErr := ResolveTerraformModules(context.Background(), configPaths, mockOptions, nil, mockHowThesePathsWereFound) - require.Error(t, actualErr) + bRan := false + moduleB := &TerraformModule{ + Path: "b", + Dependencies: TerraformModules{moduleA}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", nil, &bRan), + } - underlying, ok := errors.Unwrap(actualErr).(ErrorProcessingModule) - require.True(t, ok) + cRan := false + moduleC := &TerraformModule{ + Path: "c", + Dependencies: TerraformModules{moduleB}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), + } + + expectedErrB := ProcessingModuleDependencyError{moduleB, moduleA, expectedErrA} + expectedErrC := ProcessingModuleDependencyError{moduleC, moduleB, expectedErrB} - unwrapped := errors.Unwrap(underlying.UnderlyingError) - assert.True(t, os.IsNotExist(unwrapped), "Expected a file not exists error but got %v", underlying.UnderlyingError) + opts, err := options.NewTerragruntOptionsForTest("") + assert.NoError(t, err) + + modules := TerraformModules{moduleA, moduleB, moduleC} + err = modules.RunModules(context.Background(), opts, options.DefaultParallelism) + assertMultiErrorContains(t, err, expectedErrA, expectedErrB, expectedErrC) + + assert.True(t, aRan) + assert.False(t, bRan) + assert.False(t, cRan) } -func TestResolveTerraformModuleNoTerraformConfig(t *testing.T) { +func TestRunModulesIgnoreOrderMultipleModulesWithDependenciesMultipleFailures(t *testing.T) { t.Parallel() - configPaths := []string{"../test/fixture-modules/module-l/" + config.DefaultTerragruntConfigPath} - expected := []*TerraformModule{} + aRan := false + expectedErrA := fmt.Errorf("Expected error for module a") + moduleA := &TerraformModule{ + Path: "a", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", expectedErrA, &aRan), + } - actualModules, actualErr := ResolveTerraformModules(context.Background(), configPaths, mockOptions, nil, mockHowThesePathsWereFound) - assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) - assertModuleListsEqual(t, expected, actualModules) -} + bRan := false + moduleB := &TerraformModule{ + Path: "b", + Dependencies: TerraformModules{moduleA}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", nil, &bRan), + } -func ptr(str string) *string { - return &str + cRan := false + moduleC := &TerraformModule{ + Path: "c", + Dependencies: TerraformModules{moduleB}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), + } + + opts, err := options.NewTerragruntOptionsForTest("") + assert.NoError(t, err) + + modules := TerraformModules{moduleA, moduleB, moduleC} + err = modules.RunModulesIgnoreOrder(context.Background(), opts, options.DefaultParallelism) + assertMultiErrorContains(t, err, expectedErrA) + + assert.True(t, aRan) + assert.True(t, bRan) + assert.True(t, cRan) } -func TestLogReductionHook(t *testing.T) { +func TestRunModulesMultipleModulesWithDependenciesLargeGraphAllSuccess(t *testing.T) { t.Parallel() - var hook = NewForceLogLevelHook(logrus.ErrorLevel) - stdout := bytes.Buffer{} + aRan := false + moduleA := &TerraformModule{ + Path: "a", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), + } - var testLogger = logrus.New() - testLogger.Out = &stdout - testLogger.AddHook(hook) - testLogger.Level = logrus.DebugLevel + bRan := false + moduleB := &TerraformModule{ + Path: "b", + Dependencies: TerraformModules{moduleA}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", nil, &bRan), + } - logrus.NewEntry(testLogger).Info("Test tomato") - logrus.NewEntry(testLogger).Error("666 potato 111") + cRan := false + moduleC := &TerraformModule{ + Path: "c", + Dependencies: TerraformModules{moduleB}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), + } - out := stdout.String() + dRan := false + moduleD := &TerraformModule{ + Path: "d", + Dependencies: TerraformModules{moduleA, moduleB, moduleC}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "d", nil, &dRan), + } - var firstLogEntry = "" - var secondLogEntry = "" + eRan := false + moduleE := &TerraformModule{ + Path: "e", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "e", nil, &eRan), + } - for _, line := range strings.Split(out, "\n") { - if strings.Contains(line, "tomato") { - firstLogEntry = line - continue - } - if strings.Contains(line, "potato") { - secondLogEntry = line - continue - } + fRan := false + moduleF := &TerraformModule{ + Path: "f", + Dependencies: TerraformModules{moduleE, moduleD}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "f", nil, &fRan), } - // check that both entries got logged with error level - assert.Contains(t, firstLogEntry, "level=error") - assert.Contains(t, secondLogEntry, "level=error") + opts, err := options.NewTerragruntOptionsForTest("") + assert.NoError(t, err) + + modules := TerraformModules{moduleA, moduleB, moduleC, moduleD, moduleE, moduleF} + err = modules.RunModules(context.Background(), opts, options.DefaultParallelism) + assert.NoError(t, err) + + assert.True(t, aRan) + assert.True(t, bRan) + assert.True(t, cRan) + assert.True(t, dRan) + assert.True(t, eRan) + assert.True(t, fRan) } -func TestBasicDependency(t *testing.T) { - moduleC := &TerraformModule{Path: "C", Dependencies: []*TerraformModule{}} - moduleB := &TerraformModule{Path: "B", Dependencies: []*TerraformModule{moduleC}} - moduleA := &TerraformModule{Path: "A", Dependencies: []*TerraformModule{moduleB}} +func TestRunModulesMultipleModulesWithDependenciesLargeGraphPartialFailure(t *testing.T) { + t.Parallel() - stack := &Stack{ - Path: "test-stack", - Modules: []*TerraformModule{moduleA, moduleB, moduleC}, + aRan := false + moduleA := &TerraformModule{ + Path: "large-graph-a", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "large-graph-a", nil, &aRan), } - expected := map[string][]string{ - "B": {"A"}, - "C": {"B", "A"}, + bRan := false + moduleB := &TerraformModule{ + Path: "large-graph-b", + Dependencies: TerraformModules{moduleA}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "large-graph-b", nil, &bRan), } - result := ListStackDependentModules(stack) - - if !reflect.DeepEqual(result, expected) { - t.Errorf("Expected %v, got %v", expected, result) + cRan := false + expectedErrC := fmt.Errorf("Expected error for module large-graph-c") + moduleC := &TerraformModule{ + Path: "large-graph-c", + Dependencies: TerraformModules{moduleB}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "large-graph-c", expectedErrC, &cRan), } -} -func TestNestedDependencies(t *testing.T) { - moduleD := &TerraformModule{Path: "D", Dependencies: []*TerraformModule{}} - moduleC := &TerraformModule{Path: "C", Dependencies: []*TerraformModule{moduleD}} - moduleB := &TerraformModule{Path: "B", Dependencies: []*TerraformModule{moduleC}} - moduleA := &TerraformModule{Path: "A", Dependencies: []*TerraformModule{moduleB}} - // Create a mock stack - stack := &Stack{ - Path: "nested-stack", - Modules: []*TerraformModule{moduleA, moduleB, moduleC, moduleD}, + dRan := false + moduleD := &TerraformModule{ + Path: "large-graph-d", + Dependencies: TerraformModules{moduleA, moduleB, moduleC}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "large-graph-d", nil, &dRan), } - // Expected result - expected := map[string][]string{ - "B": {"A"}, - "C": {"B", "A"}, - "D": {"C", "B", "A"}, + eRan := false + moduleE := &TerraformModule{ + Path: "large-graph-e", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "large-graph-e", nil, &eRan), + AssumeAlreadyApplied: true, } - // Run the function - result := ListStackDependentModules(stack) + fRan := false + moduleF := &TerraformModule{ + Path: "large-graph-f", + Dependencies: TerraformModules{moduleE, moduleD}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "large-graph-f", nil, &fRan), + } - if !reflect.DeepEqual(result, expected) { - t.Errorf("Expected %v, got %v", expected, result) + gRan := false + moduleG := &TerraformModule{ + Path: "large-graph-g", + Dependencies: TerraformModules{moduleE}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "large-graph-g", nil, &gRan), } + + expectedErrD := ProcessingModuleDependencyError{moduleD, moduleC, expectedErrC} + expectedErrF := ProcessingModuleDependencyError{moduleF, moduleD, expectedErrD} + + opts, err := options.NewTerragruntOptionsForTest("") + assert.NoError(t, err) + + modules := TerraformModules{moduleA, moduleB, moduleC, moduleD, moduleE, moduleF, moduleG} + err = modules.RunModules(context.Background(), opts, options.DefaultParallelism) + assertMultiErrorContains(t, err, expectedErrC, expectedErrD, expectedErrF) + + assert.True(t, aRan) + assert.True(t, bRan) + assert.True(t, cRan) + assert.False(t, dRan) + assert.False(t, eRan) + assert.False(t, fRan) + assert.True(t, gRan) } -func TestCircularDependencies(t *testing.T) { - // Mock modules with circular dependencies - moduleA := &TerraformModule{Path: "A"} - moduleB := &TerraformModule{Path: "B"} - moduleC := &TerraformModule{Path: "C"} +func TestRunModulesReverseOrderMultipleModulesWithDependenciesLargeGraphPartialFailure(t *testing.T) { + t.Parallel() + + aRan := false + moduleA := &TerraformModule{ + Path: "a", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), + } - moduleA.Dependencies = []*TerraformModule{moduleB} - moduleB.Dependencies = []*TerraformModule{moduleC} - moduleC.Dependencies = []*TerraformModule{moduleA} // Circular dependency + bRan := false + moduleB := &TerraformModule{ + Path: "b", + Dependencies: TerraformModules{moduleA}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", nil, &bRan), + } - stack := &Stack{ - Path: "circular-stack", - Modules: []*TerraformModule{moduleA, moduleB, moduleC}, + cRan := false + expectedErrC := fmt.Errorf("Expected error for module c") + moduleC := &TerraformModule{ + Path: "c", + Dependencies: TerraformModules{moduleB}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", expectedErrC, &cRan), } - expected := map[string][]string{ - "A": {"C", "B"}, - "B": {"A", "C"}, - "C": {"B", "A"}, + dRan := false + moduleD := &TerraformModule{ + Path: "d", + Dependencies: TerraformModules{moduleA, moduleB, moduleC}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "d", nil, &dRan), } - // Run the function - result := ListStackDependentModules(stack) + eRan := false + moduleE := &TerraformModule{ + Path: "e", + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "e", nil, &eRan), + } - if !reflect.DeepEqual(result, expected) { - t.Errorf("Expected %v, got %v", expected, result) + fRan := false + moduleF := &TerraformModule{ + Path: "f", + Dependencies: TerraformModules{moduleE, moduleD}, + Config: config.TerragruntConfig{}, + TerragruntOptions: optionsWithMockTerragruntCommand(t, "f", nil, &fRan), } + + expectedErrB := ProcessingModuleDependencyError{moduleB, moduleC, expectedErrC} + expectedErrA := ProcessingModuleDependencyError{moduleA, moduleB, expectedErrB} + + opts, optsErr := options.NewTerragruntOptionsForTest("") + assert.NoError(t, optsErr) + + modules := TerraformModules{moduleA, moduleB, moduleC, moduleD, moduleE, moduleF} + err := modules.RunModulesReverseOrder(context.Background(), opts, options.DefaultParallelism) + assertMultiErrorContains(t, err, expectedErrC, expectedErrB, expectedErrA) + + assert.False(t, aRan) + assert.False(t, bRan) + assert.True(t, cRan) + assert.True(t, dRan) + assert.True(t, eRan) + assert.True(t, fRan) } diff --git a/configstack/options.go b/configstack/options.go new file mode 100644 index 000000000..659bae56a --- /dev/null +++ b/configstack/options.go @@ -0,0 +1,22 @@ +package configstack + +import ( + "github.com/gruntwork-io/terragrunt/config" + "github.com/gruntwork-io/terragrunt/config/hclparse" +) + +type Option func(Stack) Stack + +func WithChildTerragruntConfig(config *config.TerragruntConfig) Option { + return func(stack Stack) Stack { + stack.childTerragruntConfig = config + return stack + } +} + +func WithParseOptions(parserOptions []hclparse.Option) Option { + return func(stack Stack) Stack { + stack.parserOptions = parserOptions + return stack + } +} diff --git a/configstack/running_module.go b/configstack/running_module.go index b9ba8f62f..5559250be 100644 --- a/configstack/running_module.go +++ b/configstack/running_module.go @@ -3,25 +3,18 @@ package configstack import ( "bytes" "context" - "fmt" "os" "path/filepath" + "sort" "sync" - "github.com/gruntwork-io/terragrunt/terraform" - + "github.com/gruntwork-io/go-commons/errors" "github.com/gruntwork-io/terragrunt/options" - "github.com/gruntwork-io/terragrunt/telemetry" - - "github.com/gruntwork-io/go-commons/errors" - "github.com/gruntwork-io/terragrunt/shell" + "github.com/gruntwork-io/terragrunt/terraform" "github.com/hashicorp/go-multierror" ) -// Represents the status of a module that we are trying to apply as part of the apply-all or destroy-all command -type ModuleStatus int - const ( Waiting ModuleStatus = iota Running @@ -29,6 +22,18 @@ const ( channelSize = 1000 // Use a huge buffer to ensure senders are never blocked ) +const ( + NormalOrder DependencyOrder = iota + ReverseOrder + IgnoreOrder +) + +// Represents the status of a module that we are trying to apply as part of the apply-all or destroy-all command +type ModuleStatus int + +// This controls in what order dependencies should be enforced between modules +type DependencyOrder int + // Represents a module we are trying to "run" (i.e. apply or destroy) as part of the apply-all or destroy-all command type runningModule struct { Module *TerraformModule @@ -40,15 +45,6 @@ type runningModule struct { FlagExcluded bool } -// This controls in what order dependencies should be enforced between modules -type DependencyOrder int - -const ( - NormalOrder DependencyOrder = iota - ReverseOrder - IgnoreOrder -) - // Create a new RunningModule struct for the given module. This will initialize all fields to reasonable defaults, // except for the Dependencies and NotifyWhenDone, both of which will be empty. You should fill these using a // function such as crossLinkDependencies. @@ -63,146 +59,6 @@ func newRunningModule(module *TerraformModule) *runningModule { } } -// Run the given map of module path to runningModule. To "run" a module, execute the RunTerragrunt command in its -// TerragruntOptions object. The modules will be executed in an order determined by their inter-dependencies, using -// as much concurrency as possible. -func RunModules(ctx context.Context, opts *options.TerragruntOptions, modules []*TerraformModule, parallelism int) error { - runningModules, err := toRunningModules(modules, NormalOrder) - if err != nil { - return err - } - return runModules(ctx, opts, runningModules, parallelism) -} - -// Run the given map of module path to runningModule. To "run" a module, execute the RunTerragrunt command in its -// TerragruntOptions object. The modules will be executed in the reverse order of their inter-dependencies, using -// as much concurrency as possible. -func RunModulesReverseOrder(ctx context.Context, opts *options.TerragruntOptions, modules []*TerraformModule, parallelism int) error { - runningModules, err := toRunningModules(modules, ReverseOrder) - if err != nil { - return err - } - return runModules(ctx, opts, runningModules, parallelism) -} - -// Run the given map of module path to runningModule. To "run" a module, execute the RunTerragrunt command in its -// TerragruntOptions object. The modules will be executed without caring for inter-dependencies. -func RunModulesIgnoreOrder(ctx context.Context, opts *options.TerragruntOptions, modules []*TerraformModule, parallelism int) error { - runningModules, err := toRunningModules(modules, IgnoreOrder) - if err != nil { - return err - } - return runModules(ctx, opts, runningModules, parallelism) -} - -// Convert the list of modules to a map from module path to a runningModule struct. This struct contains information -// about executing the module, such as whether it has finished running or not and any errors that happened. Note that -// this does NOT actually run the module. For that, see the RunModules method. -func toRunningModules(modules []*TerraformModule, dependencyOrder DependencyOrder) (map[string]*runningModule, error) { - runningModules := map[string]*runningModule{} - for _, module := range modules { - runningModules[module.Path] = newRunningModule(module) - } - - crossLinkedModules, err := crossLinkDependencies(runningModules, dependencyOrder) - if err != nil { - return crossLinkedModules, err - } - - return removeFlagExcluded(crossLinkedModules), nil -} - -// Loop through the map of runningModules and for each module M: -// -// - If dependencyOrder is NormalOrder, plug in all the modules M depends on into the Dependencies field and all the -// modules that depend on M into the NotifyWhenDone field. -// - If dependencyOrder is ReverseOrder, do the reverse. -// - If dependencyOrder is IgnoreOrder, do nothing. -func crossLinkDependencies(modules map[string]*runningModule, dependencyOrder DependencyOrder) (map[string]*runningModule, error) { - for _, module := range modules { - for _, dependency := range module.Module.Dependencies { - runningDependency, hasDependency := modules[dependency.Path] - if !hasDependency { - return modules, errors.WithStackTrace(DependencyNotFoundWhileCrossLinking{module, dependency}) - } - switch dependencyOrder { - case NormalOrder: - module.Dependencies[runningDependency.Module.Path] = runningDependency - runningDependency.NotifyWhenDone = append(runningDependency.NotifyWhenDone, module) - case IgnoreOrder: - // Nothing - default: - runningDependency.Dependencies[module.Module.Path] = module - module.NotifyWhenDone = append(module.NotifyWhenDone, runningDependency) - } - } - } - - return modules, nil -} - -// Return a cleaned-up map that only contains modules and dependencies that should not be excluded -func removeFlagExcluded(modules map[string]*runningModule) map[string]*runningModule { - var finalModules = make(map[string]*runningModule) - - for key, module := range modules { - - // Only add modules that should not be excluded - if !module.FlagExcluded { - finalModules[key] = &runningModule{ - Module: module.Module, - Dependencies: make(map[string]*runningModule), - DependencyDone: module.DependencyDone, - Err: module.Err, - NotifyWhenDone: module.NotifyWhenDone, - Status: module.Status, - } - - // Only add dependencies that should not be excluded - for path, dependency := range module.Dependencies { - if !dependency.FlagExcluded { - finalModules[key].Dependencies[path] = dependency - } - } - } - } - - return finalModules -} - -// Run the given map of module path to runningModule. To "run" a module, execute the RunTerragrunt command in its -// TerragruntOptions object. The modules will be executed in an order determined by their inter-dependencies, using -// as much concurrency as possible. -func runModules(ctx context.Context, opts *options.TerragruntOptions, modules map[string]*runningModule, parallelism int) error { - var waitGroup sync.WaitGroup - var semaphore = make(chan struct{}, parallelism) // Make a semaphore from a buffered channel - - for _, module := range modules { - waitGroup.Add(1) - go func(module *runningModule) { - defer waitGroup.Done() - module.runModuleWhenReady(ctx, opts, semaphore) - }(module) - } - - waitGroup.Wait() - - return collectErrors(modules) -} - -// Collect the errors from the given modules and return a single error object to represent them, or nil if no errors -// occurred -func collectErrors(modules map[string]*runningModule) error { - var result *multierror.Error - for _, module := range modules { - if module.Err != nil { - result = multierror.Append(result, module.Err) - } - } - - return result.ErrorOrNil() -} - // Run a module once all of its dependencies have finished executing. func (module *runningModule) runModuleWhenReady(ctx context.Context, opts *options.TerragruntOptions, semaphore chan struct{}) { @@ -241,7 +97,7 @@ func (module *runningModule) waitForDependencies() error { module.Module.TerragruntOptions.Logger.Errorf("Dependency %s of module %s just finished with an error. Module %s will have to return an error too. However, because of --terragrunt-ignore-dependency-errors, module %s will run anyway.", doneDependency.Module.Path, module.Module.Path, module.Module.Path, module.Module.Path) } else { module.Module.TerragruntOptions.Logger.Errorf("Dependency %s of module %s just finished with an error. Module %s will have to return an error too.", doneDependency.Module.Path, module.Module.Path, module.Module.Path) - return DependencyFinishedWithError{module.Module, doneDependency.Module, doneDependency.Err} + return ProcessingModuleDependencyError{module.Module, doneDependency.Module, doneDependency.Err} } } else { module.Module.TerragruntOptions.Logger.Debugf("Dependency %s of module %s just finished successfully. Module %s must wait on %d more dependencies.", doneDependency.Module.Path, module.Module.Path, module.Module.Path, len(module.Dependencies)) @@ -264,7 +120,7 @@ func (module *runningModule) runNow(ctx context.Context, rootOptions *options.Te return err } // convert terragrunt output to json - if outputJsonFile(module.Module.TerragruntOptions, module.Module) != "" { + if module.Module.outputJsonFile(module.Module.TerragruntOptions) != "" { jsonOptions := module.Module.TerragruntOptions.Clone(module.Module.TerragruntOptions.TerragruntConfigPath) stdout := bytes.Buffer{} jsonOptions.IncludeModulePrefix = false @@ -272,12 +128,12 @@ func (module *runningModule) runNow(ctx context.Context, rootOptions *options.Te jsonOptions.OutputPrefix = "" jsonOptions.Writer = &stdout jsonOptions.TerraformCommand = terraform.CommandNameShow - jsonOptions.TerraformCliArgs = []string{terraform.CommandNameShow, "-json", modulePlanFile(rootOptions, module.Module)} + jsonOptions.TerraformCliArgs = []string{terraform.CommandNameShow, "-json", module.Module.planFile(rootOptions)} if err := jsonOptions.RunTerragrunt(ctx, jsonOptions); err != nil { return err } // save the json output to the file plan file - outputFile := outputJsonFile(rootOptions, module.Module) + outputFile := module.Module.outputJsonFile(rootOptions) jsonDir := filepath.Dir(outputFile) if err := os.MkdirAll(jsonDir, os.ModePerm); err != nil { return err @@ -306,30 +162,154 @@ func (module *runningModule) moduleFinished(moduleErr error) { } } -// Custom error types +type runningModules map[string]*runningModule + +func (modules runningModules) toTerraformModuleGroups(maxDepth int) []TerraformModules { + // Walk the graph in run order, capturing which groups will run at each iteration. In each iteration, this pops out + // the modules that have no dependencies and captures that as a run group. + groups := []TerraformModules{} + + for len(modules) > 0 && len(groups) < maxDepth { + currentIterationDeploy := TerraformModules{} + + // next tracks which modules are being deferred to a later run. + next := runningModules{} + // removeDep tracks which modules are run in the current iteration so that they need to be removed in the + // dependency list for the next iteration. This is separately tracked from currentIterationDeploy for + // convenience: this tracks the map key of the Dependencies attribute. + var removeDep []string + + // Iterate the modules, looking for those that have no dependencies and select them for "running". In the + // process, track those that still need to run in a separate map for further processing. + for path, module := range modules { + // Anything that is already applied is culled from the graph when running, so we ignore them here as well. + switch { + case module.Module.AssumeAlreadyApplied: + removeDep = append(removeDep, path) + case len(module.Dependencies) == 0: + currentIterationDeploy = append(currentIterationDeploy, module.Module) + removeDep = append(removeDep, path) + default: + next[path] = module + } + } + + // Go through the remaining module and remove the dependencies that were selected to run in this current + // iteration. + for _, module := range next { + for _, path := range removeDep { + _, hasDep := module.Dependencies[path] + if hasDep { + delete(module.Dependencies, path) + } + } + } + + // Sort the group by path so that it is easier to read and test. + sort.Slice( + currentIterationDeploy, + func(i, j int) bool { + return currentIterationDeploy[i].Path < currentIterationDeploy[j].Path + }, + ) + + // Finally, update the trackers so that the next iteration runs. + modules = next + if len(currentIterationDeploy) > 0 { + groups = append(groups, currentIterationDeploy) + } + } -type DependencyFinishedWithError struct { - Module *TerraformModule - Dependency *TerraformModule - Err error + return groups } -func (err DependencyFinishedWithError) Error() string { - return fmt.Sprintf("Cannot process module %s because one of its dependencies, %s, finished with an error: %s", err.Module, err.Dependency, err.Err) +// Loop through the map of runningModules and for each module M: +// +// - If dependencyOrder is NormalOrder, plug in all the modules M depends on into the Dependencies field and all the +// modules that depend on M into the NotifyWhenDone field. +// - If dependencyOrder is ReverseOrder, do the reverse. +// - If dependencyOrder is IgnoreOrder, do nothing. +func (modules runningModules) crossLinkDependencies(dependencyOrder DependencyOrder) (runningModules, error) { + for _, module := range modules { + for _, dependency := range module.Module.Dependencies { + runningDependency, hasDependency := modules[dependency.Path] + if !hasDependency { + return modules, errors.WithStackTrace(DependencyNotFoundWhileCrossLinkingError{module, dependency}) + } + switch dependencyOrder { + case NormalOrder: + module.Dependencies[runningDependency.Module.Path] = runningDependency + runningDependency.NotifyWhenDone = append(runningDependency.NotifyWhenDone, module) + case IgnoreOrder: + // Nothing + default: + runningDependency.Dependencies[module.Module.Path] = module + module.NotifyWhenDone = append(module.NotifyWhenDone, runningDependency) + } + } + } + + return modules, nil } -func (this DependencyFinishedWithError) ExitStatus() (int, error) { - if exitCode, err := shell.GetExitCode(this.Err); err == nil { - return exitCode, nil +// Return a cleaned-up map that only contains modules and dependencies that should not be excluded +func (modules runningModules) removeFlagExcluded() map[string]*runningModule { + var finalModules = make(map[string]*runningModule) + + for key, module := range modules { + + // Only add modules that should not be excluded + if !module.FlagExcluded { + finalModules[key] = &runningModule{ + Module: module.Module, + Dependencies: make(map[string]*runningModule), + DependencyDone: module.DependencyDone, + Err: module.Err, + NotifyWhenDone: module.NotifyWhenDone, + Status: module.Status, + } + + // Only add dependencies that should not be excluded + for path, dependency := range module.Dependencies { + if !dependency.FlagExcluded { + finalModules[key].Dependencies[path] = dependency + } + } + } } - return -1, this + + return finalModules } -type DependencyNotFoundWhileCrossLinking struct { - Module *runningModule - Dependency *TerraformModule +// Run the given map of module path to runningModule. To "run" a module, execute the RunTerragrunt command in its +// TerragruntOptions object. The modules will be executed in an order determined by their inter-dependencies, using +// as much concurrency as possible. +func (modules runningModules) runModules(ctx context.Context, opts *options.TerragruntOptions, parallelism int) error { + var waitGroup sync.WaitGroup + var semaphore = make(chan struct{}, parallelism) // Make a semaphore from a buffered channel + + for _, module := range modules { + waitGroup.Add(1) + go func(module *runningModule) { + defer waitGroup.Done() + module.runModuleWhenReady(ctx, opts, semaphore) + }(module) + } + + waitGroup.Wait() + + return modules.collectErrors() } -func (err DependencyNotFoundWhileCrossLinking) Error() string { - return fmt.Sprintf("Module %v specifies a dependency on module %v, but could not find that module while cross-linking dependencies. This is most likely a bug in Terragrunt. Please report it.", err.Module, err.Dependency) +// Collect the errors from the given modules and return a single error object to represent them, or nil if no errors +// occurred +func (modules runningModules) collectErrors() error { + var result *multierror.Error + for _, module := range modules { + if module.Err != nil { + result = multierror.Append(result, module.Err) + } + } + + return result.ErrorOrNil() } diff --git a/configstack/running_module_test.go b/configstack/running_module_test.go index 8740d5696..f9429b5af 100644 --- a/configstack/running_module_test.go +++ b/configstack/running_module_test.go @@ -1,8 +1,6 @@ package configstack import ( - "context" - "fmt" "testing" "github.com/gruntwork-io/terragrunt/config" @@ -15,7 +13,7 @@ var mockOptions, _ = options.NewTerragruntOptionsForTest("running_module_test") func TestToRunningModulesNoModules(t *testing.T) { t.Parallel() - testToRunningModules(t, []*TerraformModule{}, NormalOrder, map[string]*runningModule{}) + testToRunningModules(t, TerraformModules{}, NormalOrder, runningModules{}) } func TestToRunningModulesOneModuleNoDependencies(t *testing.T) { @@ -23,7 +21,7 @@ func TestToRunningModulesOneModuleNoDependencies(t *testing.T) { moduleA := &TerraformModule{ Path: "a", - Dependencies: []*TerraformModule{}, + Dependencies: TerraformModules{}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -32,12 +30,12 @@ func TestToRunningModulesOneModuleNoDependencies(t *testing.T) { Module: moduleA, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{}, + Dependencies: runningModules{}, NotifyWhenDone: []*runningModule{}, } - modules := []*TerraformModule{moduleA} - expected := map[string]*runningModule{"a": runningModuleA} + modules := TerraformModules{moduleA} + expected := runningModules{"a": runningModuleA} testToRunningModules(t, modules, NormalOrder, expected) } @@ -47,7 +45,7 @@ func TestToRunningModulesTwoModulesNoDependencies(t *testing.T) { moduleA := &TerraformModule{ Path: "a", - Dependencies: []*TerraformModule{}, + Dependencies: TerraformModules{}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -56,13 +54,13 @@ func TestToRunningModulesTwoModulesNoDependencies(t *testing.T) { Module: moduleA, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{}, + Dependencies: runningModules{}, NotifyWhenDone: []*runningModule{}, } moduleB := &TerraformModule{ Path: "b", - Dependencies: []*TerraformModule{}, + Dependencies: TerraformModules{}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -71,12 +69,12 @@ func TestToRunningModulesTwoModulesNoDependencies(t *testing.T) { Module: moduleB, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{}, + Dependencies: runningModules{}, NotifyWhenDone: []*runningModule{}, } - modules := []*TerraformModule{moduleA, moduleB} - expected := map[string]*runningModule{"a": runningModuleA, "b": runningModuleB} + modules := TerraformModules{moduleA, moduleB} + expected := runningModules{"a": runningModuleA, "b": runningModuleB} testToRunningModules(t, modules, NormalOrder, expected) } @@ -86,7 +84,7 @@ func TestToRunningModulesTwoModulesWithDependencies(t *testing.T) { moduleA := &TerraformModule{ Path: "a", - Dependencies: []*TerraformModule{}, + Dependencies: TerraformModules{}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -95,13 +93,13 @@ func TestToRunningModulesTwoModulesWithDependencies(t *testing.T) { Module: moduleA, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{}, + Dependencies: runningModules{}, NotifyWhenDone: []*runningModule{}, } moduleB := &TerraformModule{ Path: "b", - Dependencies: []*TerraformModule{moduleA}, + Dependencies: TerraformModules{moduleA}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -110,14 +108,14 @@ func TestToRunningModulesTwoModulesWithDependencies(t *testing.T) { Module: moduleB, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{"a": runningModuleA}, + Dependencies: runningModules{"a": runningModuleA}, NotifyWhenDone: []*runningModule{}, } runningModuleA.NotifyWhenDone = []*runningModule{runningModuleB} - modules := []*TerraformModule{moduleA, moduleB} - expected := map[string]*runningModule{"a": runningModuleA, "b": runningModuleB} + modules := TerraformModules{moduleA, moduleB} + expected := runningModules{"a": runningModuleA, "b": runningModuleB} testToRunningModules(t, modules, NormalOrder, expected) } @@ -127,7 +125,7 @@ func TestToRunningModulesTwoModulesWithDependenciesReverseOrder(t *testing.T) { moduleA := &TerraformModule{ Path: "a", - Dependencies: []*TerraformModule{}, + Dependencies: TerraformModules{}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -136,13 +134,13 @@ func TestToRunningModulesTwoModulesWithDependenciesReverseOrder(t *testing.T) { Module: moduleA, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{}, + Dependencies: runningModules{}, NotifyWhenDone: []*runningModule{}, } moduleB := &TerraformModule{ Path: "b", - Dependencies: []*TerraformModule{moduleA}, + Dependencies: TerraformModules{moduleA}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -151,14 +149,14 @@ func TestToRunningModulesTwoModulesWithDependenciesReverseOrder(t *testing.T) { Module: moduleB, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{}, + Dependencies: runningModules{}, NotifyWhenDone: []*runningModule{runningModuleA}, } - runningModuleA.Dependencies = map[string]*runningModule{"b": runningModuleB} + runningModuleA.Dependencies = runningModules{"b": runningModuleB} - modules := []*TerraformModule{moduleA, moduleB} - expected := map[string]*runningModule{"a": runningModuleA, "b": runningModuleB} + modules := TerraformModules{moduleA, moduleB} + expected := runningModules{"a": runningModuleA, "b": runningModuleB} testToRunningModules(t, modules, ReverseOrder, expected) } @@ -168,7 +166,7 @@ func TestToRunningModulesTwoModulesWithDependenciesIgnoreOrder(t *testing.T) { moduleA := &TerraformModule{ Path: "a", - Dependencies: []*TerraformModule{}, + Dependencies: TerraformModules{}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -177,13 +175,13 @@ func TestToRunningModulesTwoModulesWithDependenciesIgnoreOrder(t *testing.T) { Module: moduleA, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{}, + Dependencies: runningModules{}, NotifyWhenDone: []*runningModule{}, } moduleB := &TerraformModule{ Path: "b", - Dependencies: []*TerraformModule{moduleA}, + Dependencies: TerraformModules{moduleA}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -192,12 +190,12 @@ func TestToRunningModulesTwoModulesWithDependenciesIgnoreOrder(t *testing.T) { Module: moduleB, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{}, + Dependencies: runningModules{}, NotifyWhenDone: []*runningModule{}, } - modules := []*TerraformModule{moduleA, moduleB} - expected := map[string]*runningModule{"a": runningModuleA, "b": runningModuleB} + modules := TerraformModules{moduleA, moduleB} + expected := runningModules{"a": runningModuleA, "b": runningModuleB} testToRunningModules(t, modules, IgnoreOrder, expected) } @@ -207,7 +205,7 @@ func TestToRunningModulesMultipleModulesWithAndWithoutDependencies(t *testing.T) moduleA := &TerraformModule{ Path: "a", - Dependencies: []*TerraformModule{}, + Dependencies: TerraformModules{}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -216,13 +214,13 @@ func TestToRunningModulesMultipleModulesWithAndWithoutDependencies(t *testing.T) Module: moduleA, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{}, + Dependencies: runningModules{}, NotifyWhenDone: []*runningModule{}, } moduleB := &TerraformModule{ Path: "b", - Dependencies: []*TerraformModule{moduleA}, + Dependencies: TerraformModules{moduleA}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -231,13 +229,13 @@ func TestToRunningModulesMultipleModulesWithAndWithoutDependencies(t *testing.T) Module: moduleB, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{"a": runningModuleA}, + Dependencies: runningModules{"a": runningModuleA}, NotifyWhenDone: []*runningModule{}, } moduleC := &TerraformModule{ Path: "c", - Dependencies: []*TerraformModule{moduleA}, + Dependencies: TerraformModules{moduleA}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -246,13 +244,13 @@ func TestToRunningModulesMultipleModulesWithAndWithoutDependencies(t *testing.T) Module: moduleC, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{"a": runningModuleA}, + Dependencies: runningModules{"a": runningModuleA}, NotifyWhenDone: []*runningModule{}, } moduleD := &TerraformModule{ Path: "d", - Dependencies: []*TerraformModule{moduleC}, + Dependencies: TerraformModules{moduleC}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -261,13 +259,13 @@ func TestToRunningModulesMultipleModulesWithAndWithoutDependencies(t *testing.T) Module: moduleD, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{"c": runningModuleC}, + Dependencies: runningModules{"c": runningModuleC}, NotifyWhenDone: []*runningModule{}, } moduleE := &TerraformModule{ Path: "e", - Dependencies: []*TerraformModule{moduleA, moduleB, moduleC, moduleD}, + Dependencies: TerraformModules{moduleA, moduleB, moduleC, moduleD}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -276,7 +274,7 @@ func TestToRunningModulesMultipleModulesWithAndWithoutDependencies(t *testing.T) Module: moduleE, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{ + Dependencies: runningModules{ "a": runningModuleA, "b": runningModuleB, "c": runningModuleC, @@ -290,8 +288,8 @@ func TestToRunningModulesMultipleModulesWithAndWithoutDependencies(t *testing.T) runningModuleC.NotifyWhenDone = []*runningModule{runningModuleD, runningModuleE} runningModuleD.NotifyWhenDone = []*runningModule{runningModuleE} - modules := []*TerraformModule{moduleA, moduleB, moduleC, moduleD, moduleE} - expected := map[string]*runningModule{ + modules := TerraformModules{moduleA, moduleB, moduleC, moduleD, moduleE} + expected := runningModules{ "a": runningModuleA, "b": runningModuleB, "c": runningModuleC, @@ -307,7 +305,7 @@ func TestToRunningModulesMultipleModulesWithAndWithoutDependenciesReverseOrder(t moduleA := &TerraformModule{ Path: "a", - Dependencies: []*TerraformModule{}, + Dependencies: TerraformModules{}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -316,13 +314,13 @@ func TestToRunningModulesMultipleModulesWithAndWithoutDependenciesReverseOrder(t Module: moduleA, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{}, + Dependencies: runningModules{}, NotifyWhenDone: []*runningModule{}, } moduleB := &TerraformModule{ Path: "b", - Dependencies: []*TerraformModule{moduleA}, + Dependencies: TerraformModules{moduleA}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -331,13 +329,13 @@ func TestToRunningModulesMultipleModulesWithAndWithoutDependenciesReverseOrder(t Module: moduleB, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{}, + Dependencies: runningModules{}, NotifyWhenDone: []*runningModule{runningModuleA}, } moduleC := &TerraformModule{ Path: "c", - Dependencies: []*TerraformModule{moduleA}, + Dependencies: TerraformModules{moduleA}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -346,13 +344,13 @@ func TestToRunningModulesMultipleModulesWithAndWithoutDependenciesReverseOrder(t Module: moduleC, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{}, + Dependencies: runningModules{}, NotifyWhenDone: []*runningModule{runningModuleA}, } moduleD := &TerraformModule{ Path: "d", - Dependencies: []*TerraformModule{moduleC}, + Dependencies: TerraformModules{moduleC}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -361,13 +359,13 @@ func TestToRunningModulesMultipleModulesWithAndWithoutDependenciesReverseOrder(t Module: moduleD, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{}, + Dependencies: runningModules{}, NotifyWhenDone: []*runningModule{runningModuleC}, } moduleE := &TerraformModule{ Path: "e", - Dependencies: []*TerraformModule{moduleA, moduleB, moduleC, moduleD}, + Dependencies: TerraformModules{moduleA, moduleB, moduleC, moduleD}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -376,17 +374,17 @@ func TestToRunningModulesMultipleModulesWithAndWithoutDependenciesReverseOrder(t Module: moduleE, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{}, + Dependencies: runningModules{}, NotifyWhenDone: []*runningModule{runningModuleA, runningModuleB, runningModuleC, runningModuleD}, } - runningModuleA.Dependencies = map[string]*runningModule{"b": runningModuleB, "c": runningModuleC, "e": runningModuleE} - runningModuleB.Dependencies = map[string]*runningModule{"e": runningModuleE} - runningModuleC.Dependencies = map[string]*runningModule{"d": runningModuleD, "e": runningModuleE} - runningModuleD.Dependencies = map[string]*runningModule{"e": runningModuleE} + runningModuleA.Dependencies = runningModules{"b": runningModuleB, "c": runningModuleC, "e": runningModuleE} + runningModuleB.Dependencies = runningModules{"e": runningModuleE} + runningModuleC.Dependencies = runningModules{"d": runningModuleD, "e": runningModuleE} + runningModuleD.Dependencies = runningModules{"e": runningModuleE} - modules := []*TerraformModule{moduleA, moduleB, moduleC, moduleD, moduleE} - expected := map[string]*runningModule{ + modules := TerraformModules{moduleA, moduleB, moduleC, moduleD, moduleE} + expected := runningModules{ "a": runningModuleA, "b": runningModuleB, "c": runningModuleC, @@ -402,7 +400,7 @@ func TestToRunningModulesMultipleModulesWithAndWithoutDependenciesIgnoreOrder(t moduleA := &TerraformModule{ Path: "a", - Dependencies: []*TerraformModule{}, + Dependencies: TerraformModules{}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -411,13 +409,13 @@ func TestToRunningModulesMultipleModulesWithAndWithoutDependenciesIgnoreOrder(t Module: moduleA, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{}, + Dependencies: runningModules{}, NotifyWhenDone: []*runningModule{}, } moduleB := &TerraformModule{ Path: "b", - Dependencies: []*TerraformModule{moduleA}, + Dependencies: TerraformModules{moduleA}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -426,13 +424,13 @@ func TestToRunningModulesMultipleModulesWithAndWithoutDependenciesIgnoreOrder(t Module: moduleB, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{}, + Dependencies: runningModules{}, NotifyWhenDone: []*runningModule{}, } moduleC := &TerraformModule{ Path: "c", - Dependencies: []*TerraformModule{moduleA}, + Dependencies: TerraformModules{moduleA}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -441,13 +439,13 @@ func TestToRunningModulesMultipleModulesWithAndWithoutDependenciesIgnoreOrder(t Module: moduleC, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{}, + Dependencies: runningModules{}, NotifyWhenDone: []*runningModule{}, } moduleD := &TerraformModule{ Path: "d", - Dependencies: []*TerraformModule{moduleC}, + Dependencies: TerraformModules{moduleC}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -456,13 +454,13 @@ func TestToRunningModulesMultipleModulesWithAndWithoutDependenciesIgnoreOrder(t Module: moduleD, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{}, + Dependencies: runningModules{}, NotifyWhenDone: []*runningModule{}, } moduleE := &TerraformModule{ Path: "e", - Dependencies: []*TerraformModule{moduleA, moduleB, moduleC, moduleD}, + Dependencies: TerraformModules{moduleA, moduleB, moduleC, moduleD}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -471,12 +469,12 @@ func TestToRunningModulesMultipleModulesWithAndWithoutDependenciesIgnoreOrder(t Module: moduleE, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{}, + Dependencies: runningModules{}, NotifyWhenDone: []*runningModule{}, } - modules := []*TerraformModule{moduleA, moduleB, moduleC, moduleD, moduleE} - expected := map[string]*runningModule{ + modules := TerraformModules{moduleA, moduleB, moduleC, moduleD, moduleE} + expected := runningModules{ "a": runningModuleA, "b": runningModuleB, "c": runningModuleC, @@ -487,1020 +485,19 @@ func TestToRunningModulesMultipleModulesWithAndWithoutDependenciesIgnoreOrder(t testToRunningModules(t, modules, IgnoreOrder, expected) } -func testToRunningModules(t *testing.T, modules []*TerraformModule, order DependencyOrder, expected map[string]*runningModule) { - actual, err := toRunningModules(modules, order) +func testToRunningModules(t *testing.T, modules TerraformModules, order DependencyOrder, expected runningModules) { + actual, err := modules.toRunningModules(order) if assert.Nil(t, err, "For modules %v and order %v", modules, order) { assertRunningModuleMapsEqual(t, expected, actual, true, "For modules %v and order %v", modules, order) } } -func TestRunModulesNoModules(t *testing.T) { - t.Parallel() - - opts, err := options.NewTerragruntOptionsForTest("") - assert.NoError(t, err) - - err = RunModules(context.Background(), opts, []*TerraformModule{}, options.DefaultParallelism) - assert.Nil(t, err, "Unexpected error: %v", err) -} - -func TestRunModulesOneModuleSuccess(t *testing.T) { - t.Parallel() - - aRan := false - moduleA := &TerraformModule{ - Path: "a", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), - } - - opts, err := options.NewTerragruntOptionsForTest("") - assert.NoError(t, err) - - err = RunModules(context.Background(), opts, []*TerraformModule{moduleA}, options.DefaultParallelism) - assert.Nil(t, err, "Unexpected error: %v", err) - assert.True(t, aRan) -} - -func TestRunModulesOneModuleAssumeAlreadyRan(t *testing.T) { - t.Parallel() - - aRan := false - moduleA := &TerraformModule{ - Path: "a", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), - AssumeAlreadyApplied: true, - } - - opts, err := options.NewTerragruntOptionsForTest("") - assert.NoError(t, err) - - err = RunModules(context.Background(), opts, []*TerraformModule{moduleA}, options.DefaultParallelism) - assert.Nil(t, err, "Unexpected error: %v", err) - assert.False(t, aRan) -} - -func TestRunModulesReverseOrderOneModuleSuccess(t *testing.T) { - t.Parallel() - - aRan := false - moduleA := &TerraformModule{ - Path: "a", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), - } - - opts, err := options.NewTerragruntOptionsForTest("") - assert.NoError(t, err) - - err = RunModulesReverseOrder(context.Background(), opts, []*TerraformModule{moduleA}, options.DefaultParallelism) - assert.Nil(t, err, "Unexpected error: %v", err) - assert.True(t, aRan) -} - -func TestRunModulesIgnoreOrderOneModuleSuccess(t *testing.T) { - t.Parallel() - - aRan := false - moduleA := &TerraformModule{ - Path: "a", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), - } - - opts, err := options.NewTerragruntOptionsForTest("") - assert.NoError(t, err) - - err = RunModulesIgnoreOrder(context.Background(), opts, []*TerraformModule{moduleA}, options.DefaultParallelism) - assert.Nil(t, err, "Unexpected error: %v", err) - assert.True(t, aRan) -} - -func TestRunModulesOneModuleError(t *testing.T) { - t.Parallel() - - aRan := false - expectedErrA := fmt.Errorf("Expected error for module a") - moduleA := &TerraformModule{ - Path: "a", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", expectedErrA, &aRan), - } - - opts, err := options.NewTerragruntOptionsForTest("") - assert.NoError(t, err) - - err = RunModules(context.Background(), opts, []*TerraformModule{moduleA}, options.DefaultParallelism) - assertMultiErrorContains(t, err, expectedErrA) - assert.True(t, aRan) -} - -func TestRunModulesReverseOrderOneModuleError(t *testing.T) { - t.Parallel() - - aRan := false - expectedErrA := fmt.Errorf("Expected error for module a") - moduleA := &TerraformModule{ - Path: "a", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", expectedErrA, &aRan), - } - - opts, err := options.NewTerragruntOptionsForTest("") - assert.NoError(t, err) - - err = RunModulesReverseOrder(context.Background(), opts, []*TerraformModule{moduleA}, options.DefaultParallelism) - assertMultiErrorContains(t, err, expectedErrA) - assert.True(t, aRan) -} - -func TestRunModulesIgnoreOrderOneModuleError(t *testing.T) { - t.Parallel() - - aRan := false - expectedErrA := fmt.Errorf("Expected error for module a") - moduleA := &TerraformModule{ - Path: "a", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", expectedErrA, &aRan), - } - - opts, err := options.NewTerragruntOptionsForTest("") - assert.NoError(t, err) - - err = RunModulesIgnoreOrder(context.Background(), opts, []*TerraformModule{moduleA}, options.DefaultParallelism) - assertMultiErrorContains(t, err, expectedErrA) - assert.True(t, aRan) -} - -func TestRunModulesMultipleModulesNoDependenciesSuccess(t *testing.T) { - t.Parallel() - - aRan := false - moduleA := &TerraformModule{ - Path: "a", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), - } - - bRan := false - moduleB := &TerraformModule{ - Path: "b", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", nil, &bRan), - } - - cRan := false - moduleC := &TerraformModule{ - Path: "c", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), - } - - opts, err := options.NewTerragruntOptionsForTest("") - assert.NoError(t, err) - - err = RunModules(context.Background(), opts, []*TerraformModule{moduleA, moduleB, moduleC}, options.DefaultParallelism) - assert.Nil(t, err, "Unexpected error: %v", err) - - assert.True(t, aRan) - assert.True(t, bRan) - assert.True(t, cRan) -} - -func TestRunModulesMultipleModulesNoDependenciesSuccessNoParallelism(t *testing.T) { - t.Parallel() - - aRan := false - moduleA := &TerraformModule{ - Path: "a", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), - } - - bRan := false - moduleB := &TerraformModule{ - Path: "b", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", nil, &bRan), - } - - cRan := false - moduleC := &TerraformModule{ - Path: "c", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), - } - - opts, err := options.NewTerragruntOptionsForTest("") - assert.NoError(t, err) - - err = RunModules(context.Background(), opts, []*TerraformModule{moduleA, moduleB, moduleC}, 1) - assert.Nil(t, err, "Unexpected error: %v", err) - - assert.True(t, aRan) - assert.True(t, bRan) - assert.True(t, cRan) -} - -func TestRunModulesReverseOrderMultipleModulesNoDependenciesSuccess(t *testing.T) { - t.Parallel() - - aRan := false - moduleA := &TerraformModule{ - Path: "a", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), - } - - bRan := false - moduleB := &TerraformModule{ - Path: "b", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", nil, &bRan), - } - - cRan := false - moduleC := &TerraformModule{ - Path: "c", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), - } - - opts, err := options.NewTerragruntOptionsForTest("") - assert.NoError(t, err) - - err = RunModulesReverseOrder(context.Background(), opts, []*TerraformModule{moduleA, moduleB, moduleC}, options.DefaultParallelism) - assert.Nil(t, err, "Unexpected error: %v", err) - - assert.True(t, aRan) - assert.True(t, bRan) - assert.True(t, cRan) -} - -func TestRunModulesIgnoreOrderMultipleModulesNoDependenciesSuccess(t *testing.T) { - t.Parallel() - - aRan := false - moduleA := &TerraformModule{ - Path: "a", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), - } - - bRan := false - moduleB := &TerraformModule{ - Path: "b", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", nil, &bRan), - } - - cRan := false - moduleC := &TerraformModule{ - Path: "c", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), - } - - opts, err := options.NewTerragruntOptionsForTest("") - assert.NoError(t, err) - - err = RunModulesIgnoreOrder(context.Background(), opts, []*TerraformModule{moduleA, moduleB, moduleC}, options.DefaultParallelism) - assert.Nil(t, err, "Unexpected error: %v", err) - - assert.True(t, aRan) - assert.True(t, bRan) - assert.True(t, cRan) -} - -func TestRunModulesMultipleModulesNoDependenciesOneFailure(t *testing.T) { - t.Parallel() - - aRan := false - moduleA := &TerraformModule{ - Path: "a", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), - } - - bRan := false - expectedErrB := fmt.Errorf("Expected error for module b") - moduleB := &TerraformModule{ - Path: "b", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", expectedErrB, &bRan), - } - - cRan := false - moduleC := &TerraformModule{ - Path: "c", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), - } - - opts, optsErr := options.NewTerragruntOptionsForTest("") - assert.NoError(t, optsErr) - - err := RunModules(context.Background(), opts, []*TerraformModule{moduleA, moduleB, moduleC}, options.DefaultParallelism) - assertMultiErrorContains(t, err, expectedErrB) - - assert.True(t, aRan) - assert.True(t, bRan) - assert.True(t, cRan) -} - -func TestRunModulesMultipleModulesNoDependenciesMultipleFailures(t *testing.T) { - t.Parallel() - - aRan := false - expectedErrA := fmt.Errorf("Expected error for module a") - moduleA := &TerraformModule{ - Path: "a", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", expectedErrA, &aRan), - } - - bRan := false - expectedErrB := fmt.Errorf("Expected error for module b") - moduleB := &TerraformModule{ - Path: "b", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", expectedErrB, &bRan), - } - - cRan := false - expectedErrC := fmt.Errorf("Expected error for module c") - moduleC := &TerraformModule{ - Path: "c", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", expectedErrC, &cRan), - } - - opts, err := options.NewTerragruntOptionsForTest("") - assert.NoError(t, err) - - err = RunModules(context.Background(), opts, []*TerraformModule{moduleA, moduleB, moduleC}, options.DefaultParallelism) - assertMultiErrorContains(t, err, expectedErrA, expectedErrB, expectedErrC) - - assert.True(t, aRan) - assert.True(t, bRan) - assert.True(t, cRan) -} - -func TestRunModulesMultipleModulesWithDependenciesSuccess(t *testing.T) { - t.Parallel() - - aRan := false - moduleA := &TerraformModule{ - Path: "a", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), - } - - bRan := false - moduleB := &TerraformModule{ - Path: "b", - Dependencies: []*TerraformModule{moduleA}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", nil, &bRan), - } - - cRan := false - moduleC := &TerraformModule{ - Path: "c", - Dependencies: []*TerraformModule{moduleB}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), - } - - opts, err := options.NewTerragruntOptionsForTest("") - assert.NoError(t, err) - - err = RunModules(context.Background(), opts, []*TerraformModule{moduleA, moduleB, moduleC}, options.DefaultParallelism) - assert.Nil(t, err, "Unexpected error: %v", err) - - assert.True(t, aRan) - assert.True(t, bRan) - assert.True(t, cRan) -} - -func TestRunModulesMultipleModulesWithDependenciesWithAssumeAlreadyRanSuccess(t *testing.T) { - t.Parallel() - - aRan := false - moduleA := &TerraformModule{ - Path: "a", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), - } - - bRan := false - moduleB := &TerraformModule{ - Path: "b", - Dependencies: []*TerraformModule{moduleA}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", nil, &bRan), - } - - cRan := false - moduleC := &TerraformModule{ - Path: "c", - Dependencies: []*TerraformModule{moduleB}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), - AssumeAlreadyApplied: true, - } - - dRan := false - moduleD := &TerraformModule{ - Path: "d", - Dependencies: []*TerraformModule{moduleC}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "d", nil, &dRan), - } - - opts, err := options.NewTerragruntOptionsForTest("") - assert.NoError(t, err) - - err = RunModules(context.Background(), opts, []*TerraformModule{moduleA, moduleB, moduleC, moduleD}, options.DefaultParallelism) - assert.Nil(t, err, "Unexpected error: %v", err) - - assert.True(t, aRan) - assert.True(t, bRan) - assert.False(t, cRan) - assert.True(t, dRan) -} - -func TestRunModulesReverseOrderMultipleModulesWithDependenciesSuccess(t *testing.T) { - t.Parallel() - - aRan := false - moduleA := &TerraformModule{ - Path: "a", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), - } - - bRan := false - moduleB := &TerraformModule{ - Path: "b", - Dependencies: []*TerraformModule{moduleA}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", nil, &bRan), - } - - cRan := false - moduleC := &TerraformModule{ - Path: "c", - Dependencies: []*TerraformModule{moduleB}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), - } - - opts, err := options.NewTerragruntOptionsForTest("") - assert.NoError(t, err) - - err = RunModulesReverseOrder(context.Background(), opts, []*TerraformModule{moduleA, moduleB, moduleC}, options.DefaultParallelism) - assert.Nil(t, err, "Unexpected error: %v", err) - - assert.True(t, aRan) - assert.True(t, bRan) - assert.True(t, cRan) -} - -func TestRunModulesIgnoreOrderMultipleModulesWithDependenciesSuccess(t *testing.T) { - t.Parallel() - - aRan := false - moduleA := &TerraformModule{ - Path: "a", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), - } - - bRan := false - moduleB := &TerraformModule{ - Path: "b", - Dependencies: []*TerraformModule{moduleA}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", nil, &bRan), - } - - cRan := false - moduleC := &TerraformModule{ - Path: "c", - Dependencies: []*TerraformModule{moduleB}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), - } - - opts, err := options.NewTerragruntOptionsForTest("") - assert.NoError(t, err) - - err = RunModulesIgnoreOrder(context.Background(), opts, []*TerraformModule{moduleA, moduleB, moduleC}, options.DefaultParallelism) - assert.Nil(t, err, "Unexpected error: %v", err) - - assert.True(t, aRan) - assert.True(t, bRan) - assert.True(t, cRan) -} - -func TestRunModulesMultipleModulesWithDependenciesOneFailure(t *testing.T) { - t.Parallel() - - aRan := false - moduleA := &TerraformModule{ - Path: "a", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), - } - - bRan := false - expectedErrB := fmt.Errorf("Expected error for module b") - moduleB := &TerraformModule{ - Path: "b", - Dependencies: []*TerraformModule{moduleA}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", expectedErrB, &bRan), - } - - cRan := false - moduleC := &TerraformModule{ - Path: "c", - Dependencies: []*TerraformModule{moduleB}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), - } - - expectedErrC := DependencyFinishedWithError{moduleC, moduleB, expectedErrB} - - opts, err := options.NewTerragruntOptionsForTest("") - assert.NoError(t, err) - - err = RunModules(context.Background(), opts, []*TerraformModule{moduleA, moduleB, moduleC}, options.DefaultParallelism) - assertMultiErrorContains(t, err, expectedErrB, expectedErrC) - - assert.True(t, aRan) - assert.True(t, bRan) - assert.False(t, cRan) -} - -func TestRunModulesMultipleModulesWithDependenciesOneFailureIgnoreDependencyErrors(t *testing.T) { - t.Parallel() - - aRan := false - terragruntOptionsA := optionsWithMockTerragruntCommand(t, "a", nil, &aRan) - terragruntOptionsA.IgnoreDependencyErrors = true - moduleA := &TerraformModule{ - Path: "a", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: terragruntOptionsA, - } - - bRan := false - expectedErrB := fmt.Errorf("Expected error for module b") - terragruntOptionsB := optionsWithMockTerragruntCommand(t, "b", expectedErrB, &bRan) - terragruntOptionsB.IgnoreDependencyErrors = true - moduleB := &TerraformModule{ - Path: "b", - Dependencies: []*TerraformModule{moduleA}, - Config: config.TerragruntConfig{}, - TerragruntOptions: terragruntOptionsB, - } - - cRan := false - terragruntOptionsC := optionsWithMockTerragruntCommand(t, "c", nil, &cRan) - terragruntOptionsC.IgnoreDependencyErrors = true - moduleC := &TerraformModule{ - Path: "c", - Dependencies: []*TerraformModule{moduleB}, - Config: config.TerragruntConfig{}, - TerragruntOptions: terragruntOptionsC, - } - - opts, err := options.NewTerragruntOptionsForTest("") - assert.NoError(t, err) - - err = RunModules(context.Background(), opts, []*TerraformModule{moduleA, moduleB, moduleC}, options.DefaultParallelism) - assertMultiErrorContains(t, err, expectedErrB) - - assert.True(t, aRan) - assert.True(t, bRan) - assert.True(t, cRan) -} - -func TestRunModulesReverseOrderMultipleModulesWithDependenciesOneFailure(t *testing.T) { - t.Parallel() - - aRan := false - moduleA := &TerraformModule{ - Path: "a", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), - } - - bRan := false - expectedErrB := fmt.Errorf("Expected error for module b") - moduleB := &TerraformModule{ - Path: "b", - Dependencies: []*TerraformModule{moduleA}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", expectedErrB, &bRan), - } - - cRan := false - moduleC := &TerraformModule{ - Path: "c", - Dependencies: []*TerraformModule{moduleB}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), - } - - expectedErrA := DependencyFinishedWithError{moduleA, moduleB, expectedErrB} - - opts, err := options.NewTerragruntOptionsForTest("") - assert.NoError(t, err) - - err = RunModulesReverseOrder(context.Background(), opts, []*TerraformModule{moduleA, moduleB, moduleC}, options.DefaultParallelism) - assertMultiErrorContains(t, err, expectedErrB, expectedErrA) - - assert.False(t, aRan) - assert.True(t, bRan) - assert.True(t, cRan) -} - -func TestRunModulesIgnoreOrderMultipleModulesWithDependenciesOneFailure(t *testing.T) { - t.Parallel() - - aRan := false - moduleA := &TerraformModule{ - Path: "a", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), - } - - bRan := false - expectedErrB := fmt.Errorf("Expected error for module b") - moduleB := &TerraformModule{ - Path: "b", - Dependencies: []*TerraformModule{moduleA}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", expectedErrB, &bRan), - } - - cRan := false - moduleC := &TerraformModule{ - Path: "c", - Dependencies: []*TerraformModule{moduleB}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), - } - - opts, err := options.NewTerragruntOptionsForTest("") - assert.NoError(t, err) - - err = RunModulesIgnoreOrder(context.Background(), opts, []*TerraformModule{moduleA, moduleB, moduleC}, options.DefaultParallelism) - assertMultiErrorContains(t, err, expectedErrB) - - assert.True(t, aRan) - assert.True(t, bRan) - assert.True(t, cRan) -} - -func TestRunModulesMultipleModulesWithDependenciesMultipleFailures(t *testing.T) { - t.Parallel() - - aRan := false - expectedErrA := fmt.Errorf("Expected error for module a") - moduleA := &TerraformModule{ - Path: "a", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", expectedErrA, &aRan), - } - - bRan := false - moduleB := &TerraformModule{ - Path: "b", - Dependencies: []*TerraformModule{moduleA}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", nil, &bRan), - } - - cRan := false - moduleC := &TerraformModule{ - Path: "c", - Dependencies: []*TerraformModule{moduleB}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), - } - - expectedErrB := DependencyFinishedWithError{moduleB, moduleA, expectedErrA} - expectedErrC := DependencyFinishedWithError{moduleC, moduleB, expectedErrB} - - opts, err := options.NewTerragruntOptionsForTest("") - assert.NoError(t, err) - - err = RunModules(context.Background(), opts, []*TerraformModule{moduleA, moduleB, moduleC}, options.DefaultParallelism) - assertMultiErrorContains(t, err, expectedErrA, expectedErrB, expectedErrC) - - assert.True(t, aRan) - assert.False(t, bRan) - assert.False(t, cRan) -} - -func TestRunModulesIgnoreOrderMultipleModulesWithDependenciesMultipleFailures(t *testing.T) { - t.Parallel() - - aRan := false - expectedErrA := fmt.Errorf("Expected error for module a") - moduleA := &TerraformModule{ - Path: "a", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", expectedErrA, &aRan), - } - - bRan := false - moduleB := &TerraformModule{ - Path: "b", - Dependencies: []*TerraformModule{moduleA}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", nil, &bRan), - } - - cRan := false - moduleC := &TerraformModule{ - Path: "c", - Dependencies: []*TerraformModule{moduleB}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), - } - - opts, err := options.NewTerragruntOptionsForTest("") - assert.NoError(t, err) - - err = RunModulesIgnoreOrder(context.Background(), opts, []*TerraformModule{moduleA, moduleB, moduleC}, options.DefaultParallelism) - assertMultiErrorContains(t, err, expectedErrA) - - assert.True(t, aRan) - assert.True(t, bRan) - assert.True(t, cRan) -} - -func TestRunModulesMultipleModulesWithDependenciesLargeGraphAllSuccess(t *testing.T) { - t.Parallel() - - aRan := false - moduleA := &TerraformModule{ - Path: "a", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), - } - - bRan := false - moduleB := &TerraformModule{ - Path: "b", - Dependencies: []*TerraformModule{moduleA}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", nil, &bRan), - } - - cRan := false - moduleC := &TerraformModule{ - Path: "c", - Dependencies: []*TerraformModule{moduleB}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", nil, &cRan), - } - - dRan := false - moduleD := &TerraformModule{ - Path: "d", - Dependencies: []*TerraformModule{moduleA, moduleB, moduleC}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "d", nil, &dRan), - } - - eRan := false - moduleE := &TerraformModule{ - Path: "e", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "e", nil, &eRan), - } - - fRan := false - moduleF := &TerraformModule{ - Path: "f", - Dependencies: []*TerraformModule{moduleE, moduleD}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "f", nil, &fRan), - } - - opts, err := options.NewTerragruntOptionsForTest("") - assert.NoError(t, err) - - err = RunModules(context.Background(), opts, []*TerraformModule{moduleA, moduleB, moduleC, moduleD, moduleE, moduleF}, options.DefaultParallelism) - assert.NoError(t, err) - - assert.True(t, aRan) - assert.True(t, bRan) - assert.True(t, cRan) - assert.True(t, dRan) - assert.True(t, eRan) - assert.True(t, fRan) -} - -func TestRunModulesMultipleModulesWithDependenciesLargeGraphPartialFailure(t *testing.T) { - t.Parallel() - - aRan := false - moduleA := &TerraformModule{ - Path: "large-graph-a", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "large-graph-a", nil, &aRan), - } - - bRan := false - moduleB := &TerraformModule{ - Path: "large-graph-b", - Dependencies: []*TerraformModule{moduleA}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "large-graph-b", nil, &bRan), - } - - cRan := false - expectedErrC := fmt.Errorf("Expected error for module large-graph-c") - moduleC := &TerraformModule{ - Path: "large-graph-c", - Dependencies: []*TerraformModule{moduleB}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "large-graph-c", expectedErrC, &cRan), - } - - dRan := false - moduleD := &TerraformModule{ - Path: "large-graph-d", - Dependencies: []*TerraformModule{moduleA, moduleB, moduleC}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "large-graph-d", nil, &dRan), - } - - eRan := false - moduleE := &TerraformModule{ - Path: "large-graph-e", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "large-graph-e", nil, &eRan), - AssumeAlreadyApplied: true, - } - - fRan := false - moduleF := &TerraformModule{ - Path: "large-graph-f", - Dependencies: []*TerraformModule{moduleE, moduleD}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "large-graph-f", nil, &fRan), - } - - gRan := false - moduleG := &TerraformModule{ - Path: "large-graph-g", - Dependencies: []*TerraformModule{moduleE}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "large-graph-g", nil, &gRan), - } - - expectedErrD := DependencyFinishedWithError{moduleD, moduleC, expectedErrC} - expectedErrF := DependencyFinishedWithError{moduleF, moduleD, expectedErrD} - - opts, err := options.NewTerragruntOptionsForTest("") - assert.NoError(t, err) - - err = RunModules(context.Background(), opts, []*TerraformModule{moduleA, moduleB, moduleC, moduleD, moduleE, moduleF, moduleG}, options.DefaultParallelism) - assertMultiErrorContains(t, err, expectedErrC, expectedErrD, expectedErrF) - - assert.True(t, aRan) - assert.True(t, bRan) - assert.True(t, cRan) - assert.False(t, dRan) - assert.False(t, eRan) - assert.False(t, fRan) - assert.True(t, gRan) -} - -func TestRunModulesReverseOrderMultipleModulesWithDependenciesLargeGraphPartialFailure(t *testing.T) { - t.Parallel() - - aRan := false - moduleA := &TerraformModule{ - Path: "a", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "a", nil, &aRan), - } - - bRan := false - moduleB := &TerraformModule{ - Path: "b", - Dependencies: []*TerraformModule{moduleA}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "b", nil, &bRan), - } - - cRan := false - expectedErrC := fmt.Errorf("Expected error for module c") - moduleC := &TerraformModule{ - Path: "c", - Dependencies: []*TerraformModule{moduleB}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "c", expectedErrC, &cRan), - } - - dRan := false - moduleD := &TerraformModule{ - Path: "d", - Dependencies: []*TerraformModule{moduleA, moduleB, moduleC}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "d", nil, &dRan), - } - - eRan := false - moduleE := &TerraformModule{ - Path: "e", - Dependencies: []*TerraformModule{}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "e", nil, &eRan), - } - - fRan := false - moduleF := &TerraformModule{ - Path: "f", - Dependencies: []*TerraformModule{moduleE, moduleD}, - Config: config.TerragruntConfig{}, - TerragruntOptions: optionsWithMockTerragruntCommand(t, "f", nil, &fRan), - } - - expectedErrB := DependencyFinishedWithError{moduleB, moduleC, expectedErrC} - expectedErrA := DependencyFinishedWithError{moduleA, moduleB, expectedErrB} - - opts, optsErr := options.NewTerragruntOptionsForTest("") - assert.NoError(t, optsErr) - - err := RunModulesReverseOrder(context.Background(), opts, []*TerraformModule{moduleA, moduleB, moduleC, moduleD, moduleE, moduleF}, options.DefaultParallelism) - assertMultiErrorContains(t, err, expectedErrC, expectedErrB, expectedErrA) - - assert.False(t, aRan) - assert.False(t, bRan) - assert.True(t, cRan) - assert.True(t, dRan) - assert.True(t, eRan) - assert.True(t, fRan) -} - func TestRemoveFlagExcludedNoExclude(t *testing.T) { t.Parallel() moduleA := &TerraformModule{ Path: "a", - Dependencies: []*TerraformModule{}, + Dependencies: TerraformModules{}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -1509,14 +506,14 @@ func TestRemoveFlagExcludedNoExclude(t *testing.T) { Module: moduleA, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{}, + Dependencies: runningModules{}, NotifyWhenDone: []*runningModule{}, FlagExcluded: false, } moduleB := &TerraformModule{ Path: "b", - Dependencies: []*TerraformModule{moduleA}, + Dependencies: TerraformModules{moduleA}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -1525,14 +522,14 @@ func TestRemoveFlagExcludedNoExclude(t *testing.T) { Module: moduleB, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{"a": runningModuleA}, + Dependencies: runningModules{"a": runningModuleA}, NotifyWhenDone: []*runningModule{}, FlagExcluded: false, } moduleC := &TerraformModule{ Path: "c", - Dependencies: []*TerraformModule{moduleA}, + Dependencies: TerraformModules{moduleA}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -1541,14 +538,14 @@ func TestRemoveFlagExcludedNoExclude(t *testing.T) { Module: moduleC, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{"a": runningModuleA}, + Dependencies: runningModules{"a": runningModuleA}, NotifyWhenDone: []*runningModule{}, FlagExcluded: false, } moduleD := &TerraformModule{ Path: "d", - Dependencies: []*TerraformModule{moduleC}, + Dependencies: TerraformModules{moduleC}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -1557,14 +554,14 @@ func TestRemoveFlagExcludedNoExclude(t *testing.T) { Module: moduleD, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{"c": runningModuleC}, + Dependencies: runningModules{"c": runningModuleC}, NotifyWhenDone: []*runningModule{}, FlagExcluded: false, } moduleE := &TerraformModule{ Path: "e", - Dependencies: []*TerraformModule{moduleB, moduleD}, + Dependencies: TerraformModules{moduleB, moduleD}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -1573,7 +570,7 @@ func TestRemoveFlagExcludedNoExclude(t *testing.T) { Module: moduleE, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{ + Dependencies: runningModules{ "b": runningModuleB, "d": runningModuleD, }, @@ -1581,7 +578,7 @@ func TestRemoveFlagExcludedNoExclude(t *testing.T) { FlagExcluded: false, } - running_modules := map[string]*runningModule{ + running_modules := runningModules{ "a": runningModuleA, "b": runningModuleB, "c": runningModuleC, @@ -1589,7 +586,7 @@ func TestRemoveFlagExcludedNoExclude(t *testing.T) { "e": runningModuleE, } - expected := map[string]*runningModule{ + expected := runningModules{ "a": runningModuleA, "b": runningModuleB, "c": runningModuleC, @@ -1597,7 +594,7 @@ func TestRemoveFlagExcludedNoExclude(t *testing.T) { "e": runningModuleE, } - actual := removeFlagExcluded(running_modules) + actual := running_modules.removeFlagExcluded() assertRunningModuleMapsEqual(t, expected, actual, true) } @@ -1606,7 +603,7 @@ func TestRemoveFlagExcludedOneExcludeNoDependencies(t *testing.T) { moduleA := &TerraformModule{ Path: "a", - Dependencies: []*TerraformModule{}, + Dependencies: TerraformModules{}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -1615,14 +612,14 @@ func TestRemoveFlagExcludedOneExcludeNoDependencies(t *testing.T) { Module: moduleA, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{}, + Dependencies: runningModules{}, NotifyWhenDone: []*runningModule{}, FlagExcluded: false, } moduleB := &TerraformModule{ Path: "b", - Dependencies: []*TerraformModule{moduleA}, + Dependencies: TerraformModules{moduleA}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -1631,14 +628,14 @@ func TestRemoveFlagExcludedOneExcludeNoDependencies(t *testing.T) { Module: moduleB, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{"a": runningModuleA}, + Dependencies: runningModules{"a": runningModuleA}, NotifyWhenDone: []*runningModule{}, FlagExcluded: false, } moduleC := &TerraformModule{ Path: "c", - Dependencies: []*TerraformModule{moduleA}, + Dependencies: TerraformModules{moduleA}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -1647,23 +644,23 @@ func TestRemoveFlagExcludedOneExcludeNoDependencies(t *testing.T) { Module: moduleC, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{"a": runningModuleA}, + Dependencies: runningModules{"a": runningModuleA}, NotifyWhenDone: []*runningModule{}, FlagExcluded: true, } - running_modules := map[string]*runningModule{ + running_modules := runningModules{ "a": runningModuleA, "b": runningModuleB, "c": runningModuleC, } - expected := map[string]*runningModule{ + expected := runningModules{ "a": runningModuleA, "b": runningModuleB, } - actual := removeFlagExcluded(running_modules) + actual := running_modules.removeFlagExcluded() assertRunningModuleMapsEqual(t, expected, actual, true) } @@ -1672,7 +669,7 @@ func TestRemoveFlagExcludedOneExcludeWithDependencies(t *testing.T) { moduleA := &TerraformModule{ Path: "a", - Dependencies: []*TerraformModule{}, + Dependencies: TerraformModules{}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -1681,14 +678,14 @@ func TestRemoveFlagExcludedOneExcludeWithDependencies(t *testing.T) { Module: moduleA, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{}, + Dependencies: runningModules{}, NotifyWhenDone: []*runningModule{}, FlagExcluded: false, } moduleB := &TerraformModule{ Path: "b", - Dependencies: []*TerraformModule{moduleA}, + Dependencies: TerraformModules{moduleA}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -1697,14 +694,14 @@ func TestRemoveFlagExcludedOneExcludeWithDependencies(t *testing.T) { Module: moduleB, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{"a": runningModuleA}, + Dependencies: runningModules{"a": runningModuleA}, NotifyWhenDone: []*runningModule{}, FlagExcluded: false, } moduleC := &TerraformModule{ Path: "c", - Dependencies: []*TerraformModule{moduleA}, + Dependencies: TerraformModules{moduleA}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -1713,14 +710,14 @@ func TestRemoveFlagExcludedOneExcludeWithDependencies(t *testing.T) { Module: moduleC, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{"a": runningModuleA}, + Dependencies: runningModules{"a": runningModuleA}, NotifyWhenDone: []*runningModule{}, FlagExcluded: true, } moduleD := &TerraformModule{ Path: "d", - Dependencies: []*TerraformModule{moduleB, moduleC}, + Dependencies: TerraformModules{moduleB, moduleC}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -1729,7 +726,7 @@ func TestRemoveFlagExcludedOneExcludeWithDependencies(t *testing.T) { Module: moduleD, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{ + Dependencies: runningModules{ "b": runningModuleB, "c": runningModuleC, }, @@ -1739,7 +736,7 @@ func TestRemoveFlagExcludedOneExcludeWithDependencies(t *testing.T) { moduleE := &TerraformModule{ Path: "e", - Dependencies: []*TerraformModule{moduleB, moduleD}, + Dependencies: TerraformModules{moduleB, moduleD}, Config: config.TerragruntConfig{}, TerragruntOptions: mockOptions, } @@ -1748,7 +745,7 @@ func TestRemoveFlagExcludedOneExcludeWithDependencies(t *testing.T) { Module: moduleE, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{ + Dependencies: runningModules{ "b": runningModuleB, "d": runningModuleD, }, @@ -1756,25 +753,25 @@ func TestRemoveFlagExcludedOneExcludeWithDependencies(t *testing.T) { FlagExcluded: false, } - running_modules := map[string]*runningModule{ + running_modules := runningModules{ "a": runningModuleA, "b": runningModuleB, "c": runningModuleC, "d": runningModuleD, "e": runningModuleE, } - actual := removeFlagExcluded(running_modules) + actual := running_modules.removeFlagExcluded() _runningModuleD := &runningModule{ Module: moduleD, Status: Waiting, Err: nil, - Dependencies: map[string]*runningModule{"b": runningModuleB}, + Dependencies: runningModules{"b": runningModuleB}, NotifyWhenDone: []*runningModule{}, FlagExcluded: false, } - expected := map[string]*runningModule{ + expected := runningModules{ "a": runningModuleA, "b": runningModuleB, "d": _runningModuleD, diff --git a/configstack/stack.go b/configstack/stack.go index 4294c568d..f7c2dfdf1 100644 --- a/configstack/stack.go +++ b/configstack/stack.go @@ -13,6 +13,7 @@ import ( "github.com/gruntwork-io/go-commons/collections" + "github.com/gruntwork-io/terragrunt/config/hclparse" "github.com/gruntwork-io/terragrunt/telemetry" "github.com/gruntwork-io/terragrunt/terraform" @@ -26,8 +27,51 @@ import ( // Represents a stack of Terraform modules (i.e. folders with Terraform templates) that you can "spin up" or // "spin down" in a single command type Stack struct { - Path string - Modules []*TerraformModule + parserOptions []hclparse.Option + terragruntOptions *options.TerragruntOptions + childTerragruntConfig *config.TerragruntConfig + Modules TerraformModules +} + +// Find all the Terraform modules in the subfolders of the working directory of the given TerragruntOptions and +// assemble them into a Stack object that can be applied or destroyed in a single command +func FindStackInSubfolders(ctx context.Context, terragruntOptions *options.TerragruntOptions, opts ...Option) (*Stack, error) { + var terragruntConfigFiles []string + + err := telemetry.Telemetry(ctx, terragruntOptions, "find_files_in_path", map[string]interface{}{ + "working_dir": terragruntOptions.WorkingDir, + }, func(childCtx context.Context) error { + result, err := config.FindConfigFilesInPath(terragruntOptions.WorkingDir, terragruntOptions) + if err != nil { + return err + } + terragruntConfigFiles = result + return nil + }) + if err != nil { + return nil, err + } + + stack := NewStack(terragruntOptions, opts...) + if err := stack.createStackForTerragruntConfigPaths(ctx, terragruntConfigFiles); err != nil { + return nil, err + } + return stack, nil +} + +func NewStack(terragruntOptions *options.TerragruntOptions, opts ...Option) *Stack { + stack := &Stack{ + terragruntOptions: terragruntOptions, + parserOptions: config.DefaultParserOptions(terragruntOptions), + } + return stack.WithOptions(opts...) +} + +func (stack *Stack) WithOptions(opts ...Option) *Stack { + for _, opt := range opts { + *stack = opt(*stack) + } + return stack } // Render this stack as a human-readable string @@ -37,14 +81,14 @@ func (stack *Stack) String() string { modules = append(modules, fmt.Sprintf(" => %s", module.String())) } sort.Strings(modules) - return fmt.Sprintf("Stack at %s:\n%s", stack.Path, strings.Join(modules, "\n")) + return fmt.Sprintf("Stack at %s:\n%s", stack.terragruntOptions.WorkingDir, strings.Join(modules, "\n")) } // LogModuleDeployOrder will log the modules that will be deployed by this operation, in the order that the operations // happen. For plan and apply, the order will be bottom to top (dependencies first), while for destroy the order will be // in reverse. func (stack *Stack) LogModuleDeployOrder(logger *logrus.Entry, terraformCommand string) error { - outStr := fmt.Sprintf("The stack at %s will be processed in the following order for command %s:\n", stack.Path, terraformCommand) + outStr := fmt.Sprintf("The stack at %s will be processed in the following order for command %s:\n", stack.terragruntOptions.WorkingDir, terraformCommand) runGraph, err := stack.getModuleRunGraph(terraformCommand) if err != nil { return err @@ -86,7 +130,7 @@ func (stack *Stack) JsonModuleDeployOrder(terraformCommand string) (string, erro // Graph creates a graphviz representation of the modules func (stack *Stack) Graph(terragruntOptions *options.TerragruntOptions) { - err := WriteDot(terragruntOptions.Writer, terragruntOptions, stack.Modules) + err := stack.Modules.WriteDot(terragruntOptions.Writer, terragruntOptions) if err != nil { terragruntOptions.Logger.Warnf("Failed to graph dot: %v", err) } @@ -98,7 +142,7 @@ func (stack *Stack) Run(ctx context.Context, terragruntOptions *options.Terragru // prepare folder for output hierarchy if output folder is set if terragruntOptions.OutputFolder != "" { for _, module := range stack.Modules { - planFile := outputFile(terragruntOptions, module) + planFile := module.outputFile(terragruntOptions) planDir := filepath.Dir(planFile) if err := os.MkdirAll(planDir, os.ModePerm); err != nil { return err @@ -143,11 +187,11 @@ func (stack *Stack) Run(ctx context.Context, terragruntOptions *options.Terragru switch { case terragruntOptions.IgnoreDependencyOrder: - return RunModulesIgnoreOrder(ctx, terragruntOptions, stack.Modules, terragruntOptions.Parallelism) + return stack.Modules.RunModulesIgnoreOrder(ctx, terragruntOptions, terragruntOptions.Parallelism) case stackCmd == terraform.CommandNameDestroy: - return RunModulesReverseOrder(ctx, terragruntOptions, stack.Modules, terragruntOptions.Parallelism) + return stack.Modules.RunModulesReverseOrder(ctx, terragruntOptions, terragruntOptions.Parallelism) default: - return RunModules(ctx, terragruntOptions, stack.Modules, terragruntOptions.Parallelism) + return stack.Modules.RunModules(ctx, terragruntOptions, terragruntOptions.Parallelism) } } @@ -180,40 +224,12 @@ func (stack *Stack) summarizePlanAllErrors(terragruntOptions *options.Terragrunt } } -// Return an error if there is a dependency cycle in the modules of this stack. -func (stack *Stack) CheckForCycles() error { - return CheckForCycles(stack.Modules) -} - -// Find all the Terraform modules in the subfolders of the working directory of the given TerragruntOptions and -// assemble them into a Stack object that can be applied or destroyed in a single command -func FindStackInSubfolders(ctx context.Context, terragruntOptions *options.TerragruntOptions, childTerragruntConfig *config.TerragruntConfig) (*Stack, error) { - var terragruntConfigFiles []string - - err := telemetry.Telemetry(ctx, terragruntOptions, "find_files_in_path", map[string]interface{}{ - "working_dir": terragruntOptions.WorkingDir, - }, func(childCtx context.Context) error { - result, err := config.FindConfigFilesInPath(terragruntOptions.WorkingDir, terragruntOptions) - if err != nil { - return err - } - terragruntConfigFiles = result - return nil - }) - if err != nil { - return nil, err - } - - howThesePathsWereFound := fmt.Sprintf("Terragrunt config file found in a subdirectory of %s", terragruntOptions.WorkingDir) - return createStackForTerragruntConfigPaths(ctx, terragruntOptions.WorkingDir, terragruntConfigFiles, terragruntOptions, childTerragruntConfig, howThesePathsWereFound) -} - // Sync the TerraformCliArgs for each module in the stack to match the provided terragruntOptions struct. func (stack *Stack) syncTerraformCliArgs(terragruntOptions *options.TerragruntOptions) { for _, module := range stack.Modules { module.TerragruntOptions.TerraformCliArgs = collections.MakeCopyOfList(terragruntOptions.TerraformCliArgs) - planFile := modulePlanFile(terragruntOptions, module) + planFile := module.planFile(terragruntOptions) if planFile != "" { terragruntOptions.Logger.Debugf("Using output file %s for module %s", planFile, module.TerragruntOptions.TerragruntConfigPath) @@ -227,166 +243,459 @@ func (stack *Stack) syncTerraformCliArgs(terragruntOptions *options.TerragruntOp } } -// modulePlanFile - return plan file location, if output folder is set -func modulePlanFile(terragruntOptions *options.TerragruntOptions, module *TerraformModule) string { - planFile := "" - - // set plan file location if output folder is set - planFile = outputFile(terragruntOptions, module) - - planCommand := module.TerragruntOptions.TerraformCommand == terraform.CommandNamePlan || module.TerragruntOptions.TerraformCommand == terraform.CommandNameShow - - // in case if JSON output is enabled, and not specified planFile, save plan in working dir - if planCommand && planFile == "" && module.TerragruntOptions.JsonOutputFolder != "" { - planFile = terraform.TerraformPlanFile - } - return planFile -} - -// outputFile - return plan file location, if output folder is set -func outputFile(opts *options.TerragruntOptions, module *TerraformModule) string { - planFile := "" - if opts.OutputFolder != "" { - path, _ := filepath.Rel(opts.WorkingDir, module.Path) - dir := filepath.Join(opts.OutputFolder, path) - planFile = filepath.Join(dir, terraform.TerraformPlanFile) - } - return planFile -} - -// outputJsonFile - return plan JSON file location, if JSON output folder is set -func outputJsonFile(opts *options.TerragruntOptions, module *TerraformModule) string { - jsonPlanFile := "" - if opts.JsonOutputFolder != "" { - path, _ := filepath.Rel(opts.WorkingDir, module.Path) - dir := filepath.Join(opts.JsonOutputFolder, path) - jsonPlanFile = filepath.Join(dir, terraform.TerraformPlanJsonFile) +func (stack *Stack) toRunningModules(terraformCommand string) (runningModules, error) { + switch terraformCommand { + case terraform.CommandNameDestroy: + return stack.Modules.toRunningModules(ReverseOrder) + default: + return stack.Modules.toRunningModules(NormalOrder) } - return jsonPlanFile } // getModuleRunGraph converts the module list to a graph that shows the order in which the modules will be // applied/destroyed. The return structure is a list of lists, where the nested list represents modules that can be // deployed concurrently, and the outer list indicates the order. This will only include those modules that do NOT have // the exclude flag set. -func (stack *Stack) getModuleRunGraph(terraformCommand string) ([][]*TerraformModule, error) { - var moduleRunGraph map[string]*runningModule - var graphErr error - switch terraformCommand { - case terraform.CommandNameDestroy: - moduleRunGraph, graphErr = toRunningModules(stack.Modules, ReverseOrder) - default: - moduleRunGraph, graphErr = toRunningModules(stack.Modules, NormalOrder) - } - if graphErr != nil { - return nil, graphErr +func (stack *Stack) getModuleRunGraph(terraformCommand string) ([]TerraformModules, error) { + moduleRunGraph, err := stack.toRunningModules(terraformCommand) + if err != nil { + return nil, err } // Set maxDepth for the graph so that we don't get stuck in an infinite loop. const maxDepth = 1000 + groups := moduleRunGraph.toTerraformModuleGroups(maxDepth) + return groups, nil +} - // Walk the graph in run order, capturing which groups will run at each iteration. In each iteration, this pops out - // the modules that have no dependencies and captures that as a run group. - groups := [][]*TerraformModule{} - for len(moduleRunGraph) > 0 && len(groups) < maxDepth { - currentIterationDeploy := []*TerraformModule{} - - // next tracks which modules are being deferred to a later run. - next := map[string]*runningModule{} - // removeDep tracks which modules are run in the current iteration so that they need to be removed in the - // dependency list for the next iteration. This is separately tracked from currentIterationDeploy for - // convenience: this tracks the map key of the Dependencies attribute. - var removeDep []string - - // Iterate the modules, looking for those that have no dependencies and select them for "running". In the - // process, track those that still need to run in a separate map for further processing. - for path, module := range moduleRunGraph { - // Anything that is already applied is culled from the graph when running, so we ignore them here as well. - switch { - case module.Module.AssumeAlreadyApplied: - removeDep = append(removeDep, path) - case len(module.Dependencies) == 0: - currentIterationDeploy = append(currentIterationDeploy, module.Module) - removeDep = append(removeDep, path) - default: - next[path] = module - } - } +// Find all the Terraform modules in the folders that contain the given Terragrunt config files and assemble those +// modules into a Stack object that can be applied or destroyed in a single command +func (stack *Stack) createStackForTerragruntConfigPaths(ctx context.Context, terragruntConfigPaths []string) error { + err := telemetry.Telemetry(ctx, stack.terragruntOptions, "create_stack_for_terragrunt_config_paths", map[string]interface{}{ + "working_dir": stack.terragruntOptions.WorkingDir, + }, func(childCtx context.Context) error { - // Go through the remaining module and remove the dependencies that were selected to run in this current - // iteration. - for _, module := range next { - for _, path := range removeDep { - _, hasDep := module.Dependencies[path] - if hasDep { - delete(module.Dependencies, path) - } - } + if len(terragruntConfigPaths) == 0 { + return errors.WithStackTrace(NoTerraformModulesFound) } - // Sort the group by path so that it is easier to read and test. - sort.Slice( - currentIterationDeploy, - func(i, j int) bool { - return currentIterationDeploy[i].Path < currentIterationDeploy[j].Path - }, - ) + modules, err := stack.ResolveTerraformModules(ctx, terragruntConfigPaths) - // Finally, update the trackers so that the next iteration runs. - moduleRunGraph = next - if len(currentIterationDeploy) > 0 { - groups = append(groups, currentIterationDeploy) + if err != nil { + return errors.WithStackTrace(err) } + stack.Modules = modules + return nil + }) + if err != nil { + return errors.WithStackTrace(err) } - return groups, nil + err = telemetry.Telemetry(ctx, stack.terragruntOptions, "check_for_cycles", map[string]interface{}{ + "working_dir": stack.terragruntOptions.WorkingDir, + }, func(childCtx context.Context) error { + if err := stack.Modules.CheckForCycles(); err != nil { + return errors.WithStackTrace(err) + } + return nil + }) + if err != nil { + return errors.WithStackTrace(err) + } + + return nil } -// Find all the Terraform modules in the folders that contain the given Terragrunt config files and assemble those -// modules into a Stack object that can be applied or destroyed in a single command -func createStackForTerragruntConfigPaths(ctx context.Context, path string, terragruntConfigPaths []string, terragruntOptions *options.TerragruntOptions, childTerragruntConfig *config.TerragruntConfig, howThesePathsWereFound string) (*Stack, error) { - var stack *Stack - err := telemetry.Telemetry(ctx, terragruntOptions, "create_stack_for_terragrunt_config_paths", map[string]interface{}{ - "working_dir": terragruntOptions.WorkingDir, - "path": path, +// Go through each of the given Terragrunt configuration files and resolve the module that configuration file represents +// into a TerraformModule struct. Return the list of these TerraformModule structs. +func (stack *Stack) ResolveTerraformModules(ctx context.Context, terragruntConfigPaths []string) (TerraformModules, error) { + canonicalTerragruntConfigPaths, err := util.CanonicalPaths(terragruntConfigPaths, ".") + if err != nil { + return nil, err + } + + var modulesMap TerraformModulesMap + err = telemetry.Telemetry(ctx, stack.terragruntOptions, "resolve_modules", map[string]interface{}{ + "working_dir": stack.terragruntOptions.WorkingDir, }, func(childCtx context.Context) error { + howThesePathsWereFound := fmt.Sprintf("Terragrunt config file found in a subdirectory of %s", stack.terragruntOptions.WorkingDir) + result, err := stack.resolveModules(ctx, canonicalTerragruntConfigPaths, howThesePathsWereFound) + if err != nil { + return err + } + modulesMap = result + return nil + }) + if err != nil { + return nil, err + } - if len(terragruntConfigPaths) == 0 { - return errors.WithStackTrace(NoTerraformModulesFound) + var externalDependencies TerraformModulesMap + err = telemetry.Telemetry(ctx, stack.terragruntOptions, "resolve_external_dependencies_for_modules", map[string]interface{}{ + "working_dir": stack.terragruntOptions.WorkingDir, + }, func(childCtx context.Context) error { + result, err := stack.resolveExternalDependenciesForModules(ctx, modulesMap, TerraformModulesMap{}, 0) + if err != nil { + return err } + externalDependencies = result + return nil + }) + if err != nil { + return nil, err + } - modules, err := ResolveTerraformModules(ctx, terragruntConfigPaths, terragruntOptions, childTerragruntConfig, howThesePathsWereFound) + var crossLinkedModules TerraformModules + err = telemetry.Telemetry(ctx, stack.terragruntOptions, "crosslink_dependencies", map[string]interface{}{ + "working_dir": stack.terragruntOptions.WorkingDir, + }, func(childCtx context.Context) error { + result, err := modulesMap.mergeMaps(externalDependencies).crosslinkDependencies(canonicalTerragruntConfigPaths) if err != nil { - return errors.WithStackTrace(err) + return err } - stack = &Stack{Path: path, Modules: modules} + crossLinkedModules = result + return nil + }) + if err != nil { + return nil, err + } + var includedModules TerraformModules + err = telemetry.Telemetry(ctx, stack.terragruntOptions, "flag_included_dirs", map[string]interface{}{ + "working_dir": stack.terragruntOptions.WorkingDir, + }, func(childCtx context.Context) error { + includedModules = crossLinkedModules.flagIncludedDirs(stack.terragruntOptions) return nil }) if err != nil { - return nil, errors.WithStackTrace(err) + return nil, err } - err = telemetry.Telemetry(ctx, terragruntOptions, "check_for_cycles", map[string]interface{}{ - "working_dir": terragruntOptions.WorkingDir, - "stack_path": stack.Path, + + var includedModulesWithExcluded TerraformModules + err = telemetry.Telemetry(ctx, stack.terragruntOptions, "flag_excluded_dirs", map[string]interface{}{ + "working_dir": stack.terragruntOptions.WorkingDir, }, func(childCtx context.Context) error { - if err := stack.CheckForCycles(); err != nil { - return errors.WithStackTrace(err) + includedModulesWithExcluded = includedModules.flagExcludedDirs(stack.terragruntOptions) + return nil + }) + if err != nil { + return nil, err + } + + var finalModules TerraformModules + err = telemetry.Telemetry(ctx, stack.terragruntOptions, "flag_modules_that_dont_include", map[string]interface{}{ + "working_dir": stack.terragruntOptions.WorkingDir, + }, func(childCtx context.Context) error { + result, err := includedModulesWithExcluded.flagModulesThatDontInclude(stack.terragruntOptions) + if err != nil { + return err } + finalModules = result return nil }) if err != nil { - return nil, errors.WithStackTrace(err) + return nil, err } - return stack, nil + return finalModules, nil } -// Custom error types +// Go through each of the given Terragrunt configuration files and resolve the module that configuration file represents +// into a TerraformModule struct. Note that this method will NOT fill in the Dependencies field of the TerraformModule +// struct (see the crosslinkDependencies method for that). Return a map from module path to TerraformModule struct. +func (stack *Stack) resolveModules(ctx context.Context, canonicalTerragruntConfigPaths []string, howTheseModulesWereFound string) (TerraformModulesMap, error) { + modulesMap := TerraformModulesMap{} + for _, terragruntConfigPath := range canonicalTerragruntConfigPaths { + if !util.FileExists(terragruntConfigPath) { + return nil, ProcessingModuleError{UnderlyingError: os.ErrNotExist, ModulePath: terragruntConfigPath, HowThisModuleWasFound: howTheseModulesWereFound} + } -var NoTerraformModulesFound = fmt.Errorf("Could not find any subfolders with Terragrunt configuration files") + var module *TerraformModule + err := telemetry.Telemetry(ctx, stack.terragruntOptions, "resolve_terraform_module", map[string]interface{}{ + "config_path": terragruntConfigPath, + "working_dir": stack.terragruntOptions.WorkingDir, + }, func(childCtx context.Context) error { + m, err := stack.resolveTerraformModule(ctx, terragruntConfigPath, modulesMap, howTheseModulesWereFound) + if err != nil { + return err + } + module = m + return nil + }) + if err != nil { + return modulesMap, err + } + if module != nil { + modulesMap[module.Path] = module + var dependencies TerraformModulesMap + err := telemetry.Telemetry(ctx, stack.terragruntOptions, "resolve_dependencies_for_module", map[string]interface{}{ + "config_path": terragruntConfigPath, + "working_dir": stack.terragruntOptions.WorkingDir, + "module_path": module.Path, + }, func(childCtx context.Context) error { + deps, err := stack.resolveDependenciesForModule(ctx, module, modulesMap, true) + if err != nil { + return err + } + dependencies = deps + return nil + }) + if err != nil { + return modulesMap, err + } + modulesMap = collections.MergeMaps(modulesMap, dependencies) + } + } + + return modulesMap, nil +} + +// Create a TerraformModule struct for the Terraform module specified by the given Terragrunt configuration file path. +// Note that this method will NOT fill in the Dependencies field of the TerraformModule struct (see the +// crosslinkDependencies method for that). +func (stack *Stack) resolveTerraformModule(ctx context.Context, terragruntConfigPath string, modulesMap TerraformModulesMap, howThisModuleWasFound string) (*TerraformModule, error) { + modulePath, err := util.CanonicalPath(filepath.Dir(terragruntConfigPath), ".") + if err != nil { + return nil, err + } + + if _, ok := modulesMap[modulePath]; ok { + return nil, nil + } + + // Clone the options struct so we don't modify the original one. This is especially important as run-all operations + // happen concurrently. + opts := stack.terragruntOptions.Clone(terragruntConfigPath) + + // We need to reset the original path for each module. Otherwise, this path will be set to wherever you ran run-all + // from, which is not what any of the modules will want. + opts.OriginalTerragruntConfigPath = terragruntConfigPath + + // If `childTerragruntConfig.ProcessedIncludes` contains the path `terragruntConfigPath`, then this is a parent config + // which implies that `TerragruntConfigPath` must refer to a child configuration file, and the defined `IncludeConfig` must contain the path to the file itself + // for the built-in functions `read-terragrunt-config()`, `path_relative_to_include()` to work correctly. + var includeConfig *config.IncludeConfig + if stack.childTerragruntConfig != nil && stack.childTerragruntConfig.ProcessedIncludes.ContainsPath(terragruntConfigPath) { + includeConfig = &config.IncludeConfig{Path: terragruntConfigPath} + opts.TerragruntConfigPath = stack.terragruntOptions.OriginalTerragruntConfigPath + } + + if collections.ListContainsElement(opts.ExcludeDirs, modulePath) { + // module is excluded + return &TerraformModule{Path: modulePath, TerragruntOptions: opts, FlagExcluded: true}, nil + } + + parseCtx := config.NewParsingContext(ctx, opts). + WithParseOption(stack.parserOptions). + WithDecodeList( + // Need for initializing the modules + config.TerraformSource, + + // Need for parsing out the dependencies + config.DependenciesBlock, + config.DependencyBlock, + ) + + // We only partially parse the config, only using the pieces that we need in this section. This config will be fully + // parsed at a later stage right before the action is run. This is to delay interpolation of functions until right + // before we call out to terraform. + terragruntConfig, err := config.PartialParseConfigFile( + parseCtx, + terragruntConfigPath, + includeConfig, + ) + if err != nil { + return nil, errors.WithStackTrace(ProcessingModuleError{UnderlyingError: err, HowThisModuleWasFound: howThisModuleWasFound, ModulePath: terragruntConfigPath}) + } + + terragruntSource, err := config.GetTerragruntSourceForModule(stack.terragruntOptions.Source, modulePath, terragruntConfig) + if err != nil { + return nil, err + } + opts.Source = terragruntSource -type DependencyCycle []string + _, defaultDownloadDir, err := options.DefaultWorkingAndDownloadDirs(stack.terragruntOptions.TerragruntConfigPath) + if err != nil { + return nil, err + } + + // If we're using the default download directory, put it into the same folder as the Terragrunt configuration file. + // If we're not using the default, then the user has specified a custom download directory, and we leave it as-is. + if stack.terragruntOptions.DownloadDir == defaultDownloadDir { + _, downloadDir, err := options.DefaultWorkingAndDownloadDirs(terragruntConfigPath) + if err != nil { + return nil, err + } + stack.terragruntOptions.Logger.Debugf("Setting download directory for module %s to %s", modulePath, downloadDir) + opts.DownloadDir = downloadDir + } -func (err DependencyCycle) Error() string { - return fmt.Sprintf("Found a dependency cycle between modules: %s", strings.Join([]string(err), " -> ")) + // Fix for https://github.com/gruntwork-io/terragrunt/issues/208 + matches, err := filepath.Glob(filepath.Join(filepath.Dir(terragruntConfigPath), "*.tf")) + if err != nil { + return nil, err + } + if (terragruntConfig.Terraform == nil || terragruntConfig.Terraform.Source == nil || *terragruntConfig.Terraform.Source == "") && matches == nil { + stack.terragruntOptions.Logger.Debugf("Module %s does not have an associated terraform configuration and will be skipped.", filepath.Dir(terragruntConfigPath)) + return nil, nil + } + + if opts.IncludeModulePrefix { + opts.OutputPrefix = fmt.Sprintf("[%v] ", modulePath) + } + + return &TerraformModule{Path: modulePath, Config: *terragruntConfig, TerragruntOptions: opts}, nil +} + +// resolveDependenciesForModule looks through the dependencies of the given module and resolve the dependency paths listed in the module's config. +// If `skipExternal` is true, the func returns only dependencies that are inside of the current working directory, which means they are part of the environment the +// user is trying to apply-all or destroy-all. Note that this method will NOT fill in the Dependencies field of the TerraformModule struct (see the crosslinkDependencies method for that). +func (stack *Stack) resolveDependenciesForModule(ctx context.Context, module *TerraformModule, modulesMap TerraformModulesMap, skipExternal bool) (TerraformModulesMap, error) { + if module.Config.Dependencies == nil || len(module.Config.Dependencies.Paths) == 0 { + return TerraformModulesMap{}, nil + } + + key := fmt.Sprintf("%s-%s-%v-%v", module.Path, stack.terragruntOptions.WorkingDir, skipExternal, stack.terragruntOptions.TerraformCommand) + if value, ok := existingModules.Get(key); ok { + return *value, nil + } + + externalTerragruntConfigPaths := []string{} + for _, dependency := range module.Config.Dependencies.Paths { + dependencyPath, err := util.CanonicalPath(dependency, module.Path) + if err != nil { + return TerraformModulesMap{}, err + } + + if skipExternal && !util.HasPathPrefix(dependencyPath, stack.terragruntOptions.WorkingDir) { + continue + } + + terragruntConfigPath := config.GetDefaultConfigPath(dependencyPath) + + if _, alreadyContainsModule := modulesMap[dependencyPath]; !alreadyContainsModule { + externalTerragruntConfigPaths = append(externalTerragruntConfigPaths, terragruntConfigPath) + } + } + + howThesePathsWereFound := fmt.Sprintf("dependency of module at '%s'", module.Path) + result, err := stack.resolveModules(ctx, externalTerragruntConfigPaths, howThesePathsWereFound) + if err != nil { + return nil, err + } + + existingModules.Put(key, &result) + return result, nil +} + +// Look through the dependencies of the modules in the given map and resolve the "external" dependency paths listed in +// each modules config (i.e. those dependencies not in the given list of Terragrunt config canonical file paths). +// These external dependencies are outside of the current working directory, which means they may not be part of the +// environment the user is trying to apply-all or destroy-all. Therefore, this method also confirms whether the user wants +// to actually apply those dependencies or just assume they are already applied. Note that this method will NOT fill in +// the Dependencies field of the TerraformModule struct (see the crosslinkDependencies method for that). +func (stack *Stack) resolveExternalDependenciesForModules(ctx context.Context, modulesMap, modulesAlreadyProcessed TerraformModulesMap, recursionLevel int) (TerraformModulesMap, error) { + allExternalDependencies := TerraformModulesMap{} + modulesToSkip := modulesMap.mergeMaps(modulesAlreadyProcessed) + + // Simple protection from circular dependencies causing a Stack Overflow due to infinite recursion + if recursionLevel > maxLevelsOfRecursion { + return allExternalDependencies, errors.WithStackTrace(InfiniteRecursionError{RecursionLevel: maxLevelsOfRecursion, Modules: modulesToSkip}) + } + + sortedKeys := modulesMap.getSortedKeys() + for _, key := range sortedKeys { + module := modulesMap[key] + externalDependencies, err := stack.resolveDependenciesForModule(ctx, module, modulesToSkip, false) + if err != nil { + return externalDependencies, err + } + + for _, externalDependency := range externalDependencies { + if _, alreadyFound := modulesToSkip[externalDependency.Path]; alreadyFound { + continue + } + + shouldApply := false + if !stack.terragruntOptions.IgnoreExternalDependencies { + shouldApply, err = module.confirmShouldApplyExternalDependency(externalDependency, stack.terragruntOptions) + if err != nil { + return externalDependencies, err + } + } + + externalDependency.AssumeAlreadyApplied = !shouldApply + allExternalDependencies[externalDependency.Path] = externalDependency + } + } + + if len(allExternalDependencies) > 0 { + recursiveDependencies, err := stack.resolveExternalDependenciesForModules(ctx, allExternalDependencies, modulesMap, recursionLevel+1) + if err != nil { + return allExternalDependencies, err + } + return allExternalDependencies.mergeMaps(recursiveDependencies), nil + } + + return allExternalDependencies, nil +} + +// ListStackDependentModules - build a map with each module and its dependent modules +func (stack *Stack) ListStackDependentModules() map[string][]string { + // build map of dependent modules + // module path -> list of dependent modules + var dependentModules = make(map[string][]string) + + // build initial mapping of dependent modules + for _, module := range stack.Modules { + + if len(module.Dependencies) != 0 { + for _, dep := range module.Dependencies { + dependentModules[dep.Path] = util.RemoveDuplicatesFromList(append(dependentModules[dep.Path], module.Path)) + } + } + } + + // Floyd–Warshall inspired approach to find dependent modules + // merge map slices by key until no more updates are possible + + // Example: + // Initial setup: + // dependentModules["module1"] = ["module2", "module3"] + // dependentModules["module2"] = ["module3"] + // dependentModules["module3"] = ["module4"] + // dependentModules["module4"] = ["module5"] + + // After first iteration: (module1 += module4, module2 += module4, module3 += module5) + // dependentModules["module1"] = ["module2", "module3", "module4"] + // dependentModules["module2"] = ["module3", "module4"] + // dependentModules["module3"] = ["module4", "module5"] + // dependentModules["module4"] = ["module5"] + + // After second iteration: (module1 += module5, module2 += module5) + // dependentModules["module1"] = ["module2", "module3", "module4", "module5"] + // dependentModules["module2"] = ["module3", "module4", "module5"] + // dependentModules["module3"] = ["module4", "module5"] + // dependentModules["module4"] = ["module5"] + + // Done, no more updates and in map we have all dependent modules for each module. + + for { + noUpdates := true + for module, dependents := range dependentModules { + for _, dependent := range dependents { + initialSize := len(dependentModules[module]) + // merge without duplicates + list := util.RemoveDuplicatesFromList(append(dependentModules[module], dependentModules[dependent]...)) + list = util.RemoveElementFromList(list, module) + dependentModules[module] = list + if initialSize != len(dependentModules[module]) { + noUpdates = false + } + } + } + if noUpdates { + break + } + } + return dependentModules } diff --git a/configstack/stack_test.go b/configstack/stack_test.go index 33dfa2e9f..06225b366 100644 --- a/configstack/stack_test.go +++ b/configstack/stack_test.go @@ -4,11 +4,15 @@ import ( "context" "os" "path/filepath" + "reflect" "strings" "testing" + "github.com/gruntwork-io/go-commons/errors" + "github.com/gruntwork-io/terragrunt/codegen" "github.com/gruntwork-io/terragrunt/config" "github.com/gruntwork-io/terragrunt/options" + "github.com/gruntwork-io/terragrunt/terraform" "github.com/gruntwork-io/terragrunt/util" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -36,7 +40,7 @@ func TestFindStackInSubfolders(t *testing.T) { terragruntOptions.WorkingDir = envFolder - stack, err := FindStackInSubfolders(context.Background(), terragruntOptions, nil) + stack, err := FindStackInSubfolders(context.Background(), terragruntOptions) require.NoError(t, err) var modulePaths []string @@ -58,12 +62,12 @@ func TestGetModuleRunGraphApplyOrder(t *testing.T) { t.Parallel() stack := createTestStack() - runGraph, err := stack.getModuleRunGraph("apply") + runGraph, err := stack.getModuleRunGraph(terraform.CommandNameApply) require.NoError(t, err) assert.Equal( t, - [][]*TerraformModule{ + []TerraformModules{ { stack.Modules[1], }, @@ -83,12 +87,12 @@ func TestGetModuleRunGraphDestroyOrder(t *testing.T) { t.Parallel() stack := createTestStack() - runGraph, err := stack.getModuleRunGraph("destroy") + runGraph, err := stack.getModuleRunGraph(terraform.CommandNameDestroy) require.NoError(t, err) assert.Equal( t, - [][]*TerraformModule{ + []TerraformModules{ { stack.Modules[5], }, @@ -120,36 +124,37 @@ func createTestStack() *Stack { } vpc := &TerraformModule{ Path: filepath.Join(basePath, "vpc"), - Dependencies: []*TerraformModule{accountBaseline}, + Dependencies: TerraformModules{accountBaseline}, } lambda := &TerraformModule{ Path: filepath.Join(basePath, "lambda"), - Dependencies: []*TerraformModule{vpc}, + Dependencies: TerraformModules{vpc}, AssumeAlreadyApplied: true, } mysql := &TerraformModule{ Path: filepath.Join(basePath, "mysql"), - Dependencies: []*TerraformModule{vpc}, + Dependencies: TerraformModules{vpc}, } redis := &TerraformModule{ Path: filepath.Join(basePath, "redis"), - Dependencies: []*TerraformModule{vpc}, + Dependencies: TerraformModules{vpc}, } myapp := &TerraformModule{ Path: filepath.Join(basePath, "myapp"), - Dependencies: []*TerraformModule{mysql, redis}, + Dependencies: TerraformModules{mysql, redis}, } - return &Stack{ - Path: "/stage/mystack", - Modules: []*TerraformModule{ - accountBaseline, - vpc, - lambda, - mysql, - redis, - myapp, - }, + + stack := NewStack(&options.TerragruntOptions{WorkingDir: "/stage/mystack"}) + stack.Modules = TerraformModules{ + accountBaseline, + vpc, + lambda, + mysql, + redis, + myapp, } + + return stack } func createTempFolder(t *testing.T) string { @@ -185,3 +190,992 @@ func createDirIfNotExist(t *testing.T, path string) { } } } + +func TestResolveTerraformModulesNoPaths(t *testing.T) { + t.Parallel() + + configPaths := []string{} + expected := TerraformModules{} + stack := NewStack(mockOptions) + actualModules, actualErr := stack.ResolveTerraformModules(context.Background(), configPaths) + assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) + assertModuleListsEqual(t, expected, actualModules) +} + +func TestResolveTerraformModulesOneModuleNoDependencies(t *testing.T) { + t.Parallel() + + moduleA := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-a"), + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{ + Terraform: &config.TerraformConfig{Source: ptr("test")}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-a/"+config.DefaultTerragruntConfigPath)), + } + + configPaths := []string{"../test/fixture-modules/module-a/" + config.DefaultTerragruntConfigPath} + expected := TerraformModules{moduleA} + + stack := NewStack(mockOptions) + actualModules, actualErr := stack.ResolveTerraformModules(context.Background(), configPaths) + assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) + assertModuleListsEqual(t, expected, actualModules) +} + +func TestResolveTerraformModulesOneJsonModuleNoDependencies(t *testing.T) { + t.Parallel() + + moduleA := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/json-module-a"), + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{ + Terraform: &config.TerraformConfig{Source: ptr("test")}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/json-module-a/"+config.DefaultTerragruntJsonConfigPath)), + } + + configPaths := []string{"../test/fixture-modules/json-module-a/" + config.DefaultTerragruntJsonConfigPath} + expected := TerraformModules{moduleA} + + stack := NewStack(mockOptions) + actualModules, actualErr := stack.ResolveTerraformModules(context.Background(), configPaths) + assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) + assertModuleListsEqual(t, expected, actualModules) +} + +func TestResolveTerraformModulesOneModuleWithIncludesNoDependencies(t *testing.T) { + t.Parallel() + + moduleB := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-b/module-b-child"), + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{ + Terraform: &config.TerraformConfig{Source: ptr("...")}, + IsPartial: true, + ProcessedIncludes: map[string]config.IncludeConfig{ + "": {Path: canonical(t, "../test/fixture-modules/module-b/terragrunt.hcl")}, + }, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-b/module-b-child/"+config.DefaultTerragruntConfigPath)), + } + + configPaths := []string{"../test/fixture-modules/module-b/module-b-child/" + config.DefaultTerragruntConfigPath} + expected := TerraformModules{moduleB} + + stack := NewStack(mockOptions) + actualModules, actualErr := stack.ResolveTerraformModules(context.Background(), configPaths) + assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) + assertModuleListsEqual(t, expected, actualModules) +} + +func TestResolveTerraformModulesReadConfigFromParentConfig(t *testing.T) { + t.Parallel() + + childDir := "../test/fixture-modules/module-m/module-m-child" + childConfigPath := filepath.Join(childDir, config.DefaultTerragruntConfigPath) + + parentDir := "../test/fixture-modules/module-m" + parentCofnigPath := filepath.Join(parentDir, config.DefaultTerragruntConfigPath) + + localsConfigPaths := map[string]string{ + "env_vars": "../test/fixture-modules/module-m/env.hcl", + "tier_vars": "../test/fixture-modules/module-m/module-m-child/tier.hcl", + } + + localsConfigs := make(map[string]interface{}) + + for name, configPath := range localsConfigPaths { + opts, err := options.NewTerragruntOptionsWithConfigPath(configPath) + assert.NoError(t, err) + + ctx := config.NewParsingContext(context.Background(), opts) + cfg, err := config.PartialParseConfigFile(ctx, configPath, nil) + assert.NoError(t, err) + + localsConfigs[name] = map[string]interface{}{ + "dependencies": interface{}(nil), + "download_dir": "", + "generate": map[string]interface{}{}, + "iam_assume_role_duration": interface{}(nil), + "iam_assume_role_session_name": "", + "iam_role": "", + "iam_web_identity_token": "", + "inputs": interface{}(nil), + "locals": cfg.Locals, + "retry_max_attempts": interface{}(nil), + "retry_sleep_interval_sec": interface{}(nil), + "retryable_errors": interface{}(nil), + "skip": false, + "terraform_binary": "", + "terraform_version_constraint": "", + "terragrunt_version_constraint": "", + } + } + + moduleM := &TerraformModule{ + Path: canonical(t, childDir), + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{ + Terraform: &config.TerraformConfig{Source: ptr("...")}, + IsPartial: true, + ProcessedIncludes: map[string]config.IncludeConfig{ + "": {Path: canonical(t, "../test/fixture-modules/module-m/terragrunt.hcl")}, + }, + Locals: localsConfigs, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + FieldsMetadata: map[string]map[string]interface{}{ + "locals-env_vars": { + "found_in_file": canonical(t, "../test/fixture-modules/module-m/terragrunt.hcl"), + }, + "locals-tier_vars": { + "found_in_file": canonical(t, "../test/fixture-modules/module-m/terragrunt.hcl"), + }, + }, + }, + TerragruntOptions: mockOptions.Clone(canonical(t, childConfigPath)), + } + + configPaths := []string{childConfigPath} + childTerragruntConfig := &config.TerragruntConfig{ + ProcessedIncludes: map[string]config.IncludeConfig{ + "": { + Path: parentCofnigPath, + }, + }, + } + expected := TerraformModules{moduleM} + + mockOptions, _ := options.NewTerragruntOptionsForTest("running_module_test") + mockOptions.OriginalTerragruntConfigPath = childConfigPath + + stack := NewStack(mockOptions, WithChildTerragruntConfig(childTerragruntConfig)) + actualModules, actualErr := stack.ResolveTerraformModules(context.Background(), configPaths) + assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) + assertModuleListsEqual(t, expected, actualModules) +} + +func TestResolveTerraformModulesOneJsonModuleWithIncludesNoDependencies(t *testing.T) { + t.Parallel() + + moduleB := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/json-module-b/module-b-child"), + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{ + Terraform: &config.TerraformConfig{Source: ptr("...")}, + IsPartial: true, + ProcessedIncludes: map[string]config.IncludeConfig{ + "": {Path: canonical(t, "../test/fixture-modules/json-module-b/terragrunt.hcl")}, + }, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/json-module-b/module-b-child/"+config.DefaultTerragruntJsonConfigPath)), + } + + configPaths := []string{"../test/fixture-modules/json-module-b/module-b-child/" + config.DefaultTerragruntJsonConfigPath} + expected := TerraformModules{moduleB} + + stack := NewStack(mockOptions) + actualModules, actualErr := stack.ResolveTerraformModules(context.Background(), configPaths) + assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) + assertModuleListsEqual(t, expected, actualModules) +} + +func TestResolveTerraformModulesOneHclModuleWithIncludesNoDependencies(t *testing.T) { + t.Parallel() + + moduleB := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/hcl-module-b/module-b-child"), + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{ + Terraform: &config.TerraformConfig{Source: ptr("...")}, + IsPartial: true, + ProcessedIncludes: map[string]config.IncludeConfig{ + "": {Path: canonical(t, "../test/fixture-modules/hcl-module-b/terragrunt.hcl.json")}, + }, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/hcl-module-b/module-b-child/"+config.DefaultTerragruntConfigPath)), + } + + configPaths := []string{"../test/fixture-modules/hcl-module-b/module-b-child/" + config.DefaultTerragruntConfigPath} + expected := TerraformModules{moduleB} + + stack := NewStack(mockOptions) + actualModules, actualErr := stack.ResolveTerraformModules(context.Background(), configPaths) + assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) + assertModuleListsEqual(t, expected, actualModules) +} + +func TestResolveTerraformModulesTwoModulesWithDependencies(t *testing.T) { + t.Parallel() + + moduleA := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-a"), + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{ + Terraform: &config.TerraformConfig{Source: ptr("test")}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-a/"+config.DefaultTerragruntConfigPath)), + } + + moduleC := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-c"), + Dependencies: TerraformModules{moduleA}, + Config: config.TerragruntConfig{ + Dependencies: &config.ModuleDependencies{Paths: []string{"../module-a"}}, + Terraform: &config.TerraformConfig{Source: ptr("temp")}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-c/"+config.DefaultTerragruntConfigPath)), + } + + configPaths := []string{"../test/fixture-modules/module-a/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-c/" + config.DefaultTerragruntConfigPath} + expected := TerraformModules{moduleA, moduleC} + + stack := NewStack(mockOptions) + actualModules, actualErr := stack.ResolveTerraformModules(context.Background(), configPaths) + assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) + assertModuleListsEqual(t, expected, actualModules) +} + +func TestResolveTerraformModulesJsonModulesWithHclDependencies(t *testing.T) { + t.Parallel() + + moduleA := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-a"), + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{ + Terraform: &config.TerraformConfig{Source: ptr("test")}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-a/"+config.DefaultTerragruntConfigPath)), + } + + moduleC := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/json-module-c"), + Dependencies: TerraformModules{moduleA}, + Config: config.TerragruntConfig{ + Dependencies: &config.ModuleDependencies{Paths: []string{"../module-a"}}, + Terraform: &config.TerraformConfig{Source: ptr("temp")}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/json-module-c/"+config.DefaultTerragruntJsonConfigPath)), + } + + configPaths := []string{"../test/fixture-modules/module-a/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/json-module-c/" + config.DefaultTerragruntJsonConfigPath} + expected := TerraformModules{moduleA, moduleC} + + stack := NewStack(mockOptions) + actualModules, actualErr := stack.ResolveTerraformModules(context.Background(), configPaths) + assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) + assertModuleListsEqual(t, expected, actualModules) +} + +func TestResolveTerraformModulesHclModulesWithJsonDependencies(t *testing.T) { + t.Parallel() + + moduleA := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/json-module-a"), + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{ + Terraform: &config.TerraformConfig{Source: ptr("test")}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/json-module-a/"+config.DefaultTerragruntJsonConfigPath)), + } + + moduleC := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/hcl-module-c"), + Dependencies: TerraformModules{moduleA}, + Config: config.TerragruntConfig{ + Dependencies: &config.ModuleDependencies{Paths: []string{"../json-module-a"}}, + Terraform: &config.TerraformConfig{Source: ptr("temp")}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/hcl-module-c/"+config.DefaultTerragruntConfigPath)), + } + + configPaths := []string{"../test/fixture-modules/json-module-a/" + config.DefaultTerragruntJsonConfigPath, "../test/fixture-modules/hcl-module-c/" + config.DefaultTerragruntConfigPath} + expected := TerraformModules{moduleA, moduleC} + + stack := NewStack(mockOptions) + actualModules, actualErr := stack.ResolveTerraformModules(context.Background(), configPaths) + assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) + assertModuleListsEqual(t, expected, actualModules) +} + +func TestResolveTerraformModulesTwoModulesWithDependenciesExcludedDirsWithDependency(t *testing.T) { + t.Parallel() + + opts, _ := options.NewTerragruntOptionsForTest("running_module_test") + opts.ExcludeDirs = []string{canonical(t, "../test/fixture-modules/module-a")} + + moduleA := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-a"), + Dependencies: TerraformModules{}, + TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-a/"+config.DefaultTerragruntConfigPath)), + } + + moduleC := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-c"), + Dependencies: TerraformModules{moduleA}, + Config: config.TerragruntConfig{ + Dependencies: &config.ModuleDependencies{Paths: []string{"../module-a"}}, + Terraform: &config.TerraformConfig{Source: ptr("temp")}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-c/"+config.DefaultTerragruntConfigPath)), + } + + configPaths := []string{"../test/fixture-modules/module-a/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-c/" + config.DefaultTerragruntConfigPath} + + stack := NewStack(opts) + actualModules, actualErr := stack.ResolveTerraformModules(context.Background(), configPaths) + + // construct the expected list + moduleA.FlagExcluded = true + expected := TerraformModules{moduleA, moduleC} + + assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) + assertModuleListsEqual(t, expected, actualModules) +} + +func TestResolveTerraformModulesTwoModulesWithDependenciesExcludedDirsWithDependencyAndConflictingNaming(t *testing.T) { + t.Parallel() + + opts, _ := options.NewTerragruntOptionsForTest("running_module_test") + opts.ExcludeDirs = []string{canonical(t, "../test/fixture-modules/module-a")} + + moduleA := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-a"), + Dependencies: TerraformModules{}, + TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-a/"+config.DefaultTerragruntConfigPath)), + } + + moduleC := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-c"), + Dependencies: TerraformModules{moduleA}, + Config: config.TerragruntConfig{ + Dependencies: &config.ModuleDependencies{Paths: []string{"../module-a"}}, + Terraform: &config.TerraformConfig{Source: ptr("temp")}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-c/"+config.DefaultTerragruntConfigPath)), + } + + moduleAbba := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-abba"), + Dependencies: TerraformModules{moduleA}, + Config: config.TerragruntConfig{ + Dependencies: &config.ModuleDependencies{Paths: []string{"../module-a"}}, + Terraform: &config.TerraformConfig{Source: ptr("temp")}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-abba/"+config.DefaultTerragruntConfigPath)), + } + + configPaths := []string{"../test/fixture-modules/module-a/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-c/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-abba/" + config.DefaultTerragruntConfigPath} + + stack := NewStack(opts) + actualModules, actualErr := stack.ResolveTerraformModules(context.Background(), configPaths) + + // construct the expected list + moduleA.FlagExcluded = true + expected := TerraformModules{moduleA, moduleC, moduleAbba} + + assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) + assertModuleListsEqual(t, expected, actualModules) +} + +func TestResolveTerraformModulesTwoModulesWithDependenciesExcludedDirsWithDependencyAndConflictingNamingAndGlob(t *testing.T) { + t.Parallel() + + opts, _ := options.NewTerragruntOptionsForTest("running_module_test") + opts.ExcludeDirs = globCanonical(t, "../test/fixture-modules/module-a*") + + moduleA := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-a"), + Dependencies: TerraformModules{}, + TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-a/"+config.DefaultTerragruntConfigPath)), + } + + moduleC := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-c"), + Dependencies: TerraformModules{moduleA}, + Config: config.TerragruntConfig{ + Dependencies: &config.ModuleDependencies{Paths: []string{"../module-a"}}, + Terraform: &config.TerraformConfig{Source: ptr("temp")}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-c/"+config.DefaultTerragruntConfigPath)), + } + + moduleAbba := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-abba"), + Dependencies: TerraformModules{}, + TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-abba/"+config.DefaultTerragruntConfigPath)), + } + + configPaths := []string{"../test/fixture-modules/module-a/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-c/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-abba/" + config.DefaultTerragruntConfigPath} + + stack := NewStack(opts) + actualModules, actualErr := stack.ResolveTerraformModules(context.Background(), configPaths) + // construct the expected list + moduleA.FlagExcluded = true + moduleAbba.FlagExcluded = true + expected := TerraformModules{moduleA, moduleC, moduleAbba} + + assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) + assertModuleListsEqual(t, expected, actualModules) +} + +func TestResolveTerraformModulesTwoModulesWithDependenciesExcludedDirsWithNoDependency(t *testing.T) { + t.Parallel() + + opts, _ := options.NewTerragruntOptionsForTest("running_module_test") + opts.ExcludeDirs = []string{canonical(t, "../test/fixture-modules/module-c")} + + moduleA := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-a"), + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{ + Terraform: &config.TerraformConfig{Source: ptr("test")}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-a/"+config.DefaultTerragruntConfigPath)), + } + + moduleC := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-c"), + TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-c/"+config.DefaultTerragruntConfigPath)), + } + + configPaths := []string{"../test/fixture-modules/module-a/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-c/" + config.DefaultTerragruntConfigPath} + + stack := NewStack(opts) + actualModules, actualErr := stack.ResolveTerraformModules(context.Background(), configPaths) + + // construct the expected list + moduleC.FlagExcluded = true + expected := TerraformModules{moduleA, moduleC} + + assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) + assertModuleListsEqual(t, expected, actualModules) +} + +func TestResolveTerraformModulesTwoModulesWithDependenciesIncludedDirsWithDependency(t *testing.T) { + t.Parallel() + + opts, _ := options.NewTerragruntOptionsForTest("running_module_test") + opts.IncludeDirs = []string{canonical(t, "../test/fixture-modules/module-c")} + + moduleA := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-a"), + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{ + Terraform: &config.TerraformConfig{Source: ptr("test")}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-a/"+config.DefaultTerragruntConfigPath)), + } + + moduleC := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-c"), + Dependencies: TerraformModules{moduleA}, + Config: config.TerragruntConfig{ + Dependencies: &config.ModuleDependencies{Paths: []string{"../module-a"}}, + Terraform: &config.TerraformConfig{Source: ptr("temp")}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-c/"+config.DefaultTerragruntConfigPath)), + } + + configPaths := []string{"../test/fixture-modules/module-a/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-c/" + config.DefaultTerragruntConfigPath} + + stack := NewStack(opts) + actualModules, actualErr := stack.ResolveTerraformModules(context.Background(), configPaths) + + // construct the expected list + moduleA.FlagExcluded = false + expected := TerraformModules{moduleA, moduleC} + + assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) + assertModuleListsEqual(t, expected, actualModules) +} + +func TestResolveTerraformModulesTwoModulesWithDependenciesIncludedDirsWithNoDependency(t *testing.T) { + t.Parallel() + + opts, _ := options.NewTerragruntOptionsForTest("running_module_test") + opts.IncludeDirs = []string{canonical(t, "../test/fixture-modules/module-a")} + + moduleA := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-a"), + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{ + Terraform: &config.TerraformConfig{Source: ptr("test")}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-a/"+config.DefaultTerragruntConfigPath)), + } + + moduleC := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-c"), + Dependencies: TerraformModules{moduleA}, + Config: config.TerragruntConfig{ + Dependencies: &config.ModuleDependencies{Paths: []string{"../module-a"}}, + Terraform: &config.TerraformConfig{Source: ptr("temp")}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-c/"+config.DefaultTerragruntConfigPath)), + } + + configPaths := []string{"../test/fixture-modules/module-a/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-c/" + config.DefaultTerragruntConfigPath} + + stack := NewStack(opts) + actualModules, actualErr := stack.ResolveTerraformModules(context.Background(), configPaths) + + // construct the expected list + moduleC.FlagExcluded = true + expected := TerraformModules{moduleA, moduleC} + + assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) + assertModuleListsEqual(t, expected, actualModules) +} + +func TestResolveTerraformModulesTwoModulesWithDependenciesIncludedDirsWithDependencyExcludeModuleWithNoDependency(t *testing.T) { + t.Parallel() + + opts, _ := options.NewTerragruntOptionsForTest("running_module_test") + opts.IncludeDirs = []string{canonical(t, "../test/fixture-modules/module-c"), canonical(t, "../test/fixture-modules/module-f")} + opts.ExcludeDirs = []string{canonical(t, "../test/fixture-modules/module-f")} + + moduleA := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-a"), + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{ + Terraform: &config.TerraformConfig{Source: ptr("test")}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-a/"+config.DefaultTerragruntConfigPath)), + } + + moduleC := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-c"), + Dependencies: TerraformModules{moduleA}, + Config: config.TerragruntConfig{ + Dependencies: &config.ModuleDependencies{Paths: []string{"../module-a"}}, + Terraform: &config.TerraformConfig{Source: ptr("temp")}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: opts.Clone(canonical(t, "../test/fixture-modules/module-c/"+config.DefaultTerragruntConfigPath)), + } + + moduleF := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-f"), + Dependencies: TerraformModules{}, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-f/"+config.DefaultTerragruntConfigPath)), + AssumeAlreadyApplied: false, + } + + configPaths := []string{"../test/fixture-modules/module-a/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-c/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-f/" + config.DefaultTerragruntConfigPath} + + stack := NewStack(opts) + actualModules, actualErr := stack.ResolveTerraformModules(context.Background(), configPaths) + + // construct the expected list + moduleF.FlagExcluded = true + expected := TerraformModules{moduleA, moduleC, moduleF} + + assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) + assertModuleListsEqual(t, expected, actualModules) +} + +func TestResolveTerraformModulesMultipleModulesWithDependencies(t *testing.T) { + t.Parallel() + + moduleA := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-a"), + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{ + Terraform: &config.TerraformConfig{Source: ptr("test")}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-a/"+config.DefaultTerragruntConfigPath)), + } + + moduleB := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-b/module-b-child"), + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{ + Terraform: &config.TerraformConfig{Source: ptr("...")}, + IsPartial: true, + ProcessedIncludes: map[string]config.IncludeConfig{ + "": {Path: canonical(t, "../test/fixture-modules/module-b/terragrunt.hcl")}, + }, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-b/module-b-child/"+config.DefaultTerragruntConfigPath)), + } + + moduleC := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-c"), + Dependencies: TerraformModules{moduleA}, + Config: config.TerragruntConfig{ + Dependencies: &config.ModuleDependencies{Paths: []string{"../module-a"}}, + Terraform: &config.TerraformConfig{Source: ptr("temp")}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-c/"+config.DefaultTerragruntConfigPath)), + } + + moduleD := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-d"), + Dependencies: TerraformModules{moduleA, moduleB, moduleC}, + Config: config.TerragruntConfig{ + Dependencies: &config.ModuleDependencies{Paths: []string{"../module-a", "../module-b/module-b-child", "../module-c"}}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-d/"+config.DefaultTerragruntConfigPath)), + } + + configPaths := []string{"../test/fixture-modules/module-a/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-b/module-b-child/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-c/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-d/" + config.DefaultTerragruntConfigPath} + expected := TerraformModules{moduleA, moduleB, moduleC, moduleD} + + stack := NewStack(mockOptions) + actualModules, actualErr := stack.ResolveTerraformModules(context.Background(), configPaths) + assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) + assertModuleListsEqual(t, expected, actualModules) +} + +func TestResolveTerraformModulesMultipleModulesWithMixedDependencies(t *testing.T) { + t.Parallel() + + moduleA := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-a"), + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{ + Terraform: &config.TerraformConfig{Source: ptr("test")}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-a/"+config.DefaultTerragruntConfigPath)), + } + + moduleB := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/json-module-b/module-b-child"), + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{ + Terraform: &config.TerraformConfig{Source: ptr("...")}, + IsPartial: true, + ProcessedIncludes: map[string]config.IncludeConfig{ + "": {Path: canonical(t, "../test/fixture-modules/json-module-b/terragrunt.hcl")}, + }, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/json-module-b/module-b-child/"+config.DefaultTerragruntJsonConfigPath)), + } + + moduleC := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-c"), + Dependencies: TerraformModules{moduleA}, + Config: config.TerragruntConfig{ + Dependencies: &config.ModuleDependencies{Paths: []string{"../module-a"}}, + Terraform: &config.TerraformConfig{Source: ptr("temp")}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-c/"+config.DefaultTerragruntConfigPath)), + } + + moduleD := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/json-module-d"), + Dependencies: TerraformModules{moduleA, moduleB, moduleC}, + Config: config.TerragruntConfig{ + Dependencies: &config.ModuleDependencies{Paths: []string{"../module-a", "../json-module-b/module-b-child", "../module-c"}}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/json-module-d/"+config.DefaultTerragruntJsonConfigPath)), + } + + configPaths := []string{"../test/fixture-modules/module-a/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/json-module-b/module-b-child/" + config.DefaultTerragruntJsonConfigPath, "../test/fixture-modules/module-c/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/json-module-d/" + config.DefaultTerragruntJsonConfigPath} + expected := TerraformModules{moduleA, moduleB, moduleC, moduleD} + + stack := NewStack(mockOptions) + actualModules, actualErr := stack.ResolveTerraformModules(context.Background(), configPaths) + assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) + assertModuleListsEqual(t, expected, actualModules) +} + +func TestResolveTerraformModulesMultipleModulesWithDependenciesWithIncludes(t *testing.T) { + t.Parallel() + + moduleA := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-a"), + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{ + Terraform: &config.TerraformConfig{Source: ptr("test")}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-a/"+config.DefaultTerragruntConfigPath)), + } + + moduleB := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-b/module-b-child"), + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{ + Terraform: &config.TerraformConfig{Source: ptr("...")}, + IsPartial: true, + ProcessedIncludes: map[string]config.IncludeConfig{ + "": {Path: canonical(t, "../test/fixture-modules/module-b/terragrunt.hcl")}, + }, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-b/module-b-child/"+config.DefaultTerragruntConfigPath)), + } + + moduleE := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-e/module-e-child"), + Dependencies: TerraformModules{moduleA, moduleB}, + Config: config.TerragruntConfig{ + Dependencies: &config.ModuleDependencies{Paths: []string{"../../module-a", "../../module-b/module-b-child"}}, + Terraform: &config.TerraformConfig{Source: ptr("test")}, + IsPartial: true, + ProcessedIncludes: map[string]config.IncludeConfig{ + "": {Path: canonical(t, "../test/fixture-modules/module-e/terragrunt.hcl")}, + }, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-e/module-e-child/"+config.DefaultTerragruntConfigPath)), + } + + configPaths := []string{"../test/fixture-modules/module-a/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-b/module-b-child/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-e/module-e-child/" + config.DefaultTerragruntConfigPath} + expected := TerraformModules{moduleA, moduleB, moduleE} + + stack := NewStack(mockOptions) + actualModules, actualErr := stack.ResolveTerraformModules(context.Background(), configPaths) + assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) + assertModuleListsEqual(t, expected, actualModules) +} + +func TestResolveTerraformModulesMultipleModulesWithExternalDependencies(t *testing.T) { + t.Parallel() + + moduleF := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-f"), + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{ + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-f/"+config.DefaultTerragruntConfigPath)), + AssumeAlreadyApplied: true, + } + + moduleG := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-g"), + Dependencies: TerraformModules{moduleF}, + Config: config.TerragruntConfig{ + Dependencies: &config.ModuleDependencies{Paths: []string{"../module-f"}}, + Terraform: &config.TerraformConfig{Source: ptr("test")}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-g/"+config.DefaultTerragruntConfigPath)), + } + + configPaths := []string{"../test/fixture-modules/module-g/" + config.DefaultTerragruntConfigPath} + expected := TerraformModules{moduleF, moduleG} + + stack := NewStack(mockOptions) + actualModules, actualErr := stack.ResolveTerraformModules(context.Background(), configPaths) + assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) + assertModuleListsEqual(t, expected, actualModules) +} + +func TestResolveTerraformModulesMultipleModulesWithNestedExternalDependencies(t *testing.T) { + t.Parallel() + + moduleH := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-h"), + Dependencies: TerraformModules{}, + Config: config.TerragruntConfig{ + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-h/"+config.DefaultTerragruntConfigPath)), + AssumeAlreadyApplied: true, + } + + moduleI := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-i"), + Dependencies: TerraformModules{moduleH}, + Config: config.TerragruntConfig{ + Dependencies: &config.ModuleDependencies{Paths: []string{"../module-h"}}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-i/"+config.DefaultTerragruntConfigPath)), + AssumeAlreadyApplied: true, + } + + moduleJ := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-j"), + Dependencies: TerraformModules{moduleI}, + Config: config.TerragruntConfig{ + Dependencies: &config.ModuleDependencies{Paths: []string{"../module-i"}}, + Terraform: &config.TerraformConfig{Source: ptr("temp")}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-j/"+config.DefaultTerragruntConfigPath)), + } + + moduleK := &TerraformModule{ + Path: canonical(t, "../test/fixture-modules/module-k"), + Dependencies: TerraformModules{moduleH}, + Config: config.TerragruntConfig{ + Dependencies: &config.ModuleDependencies{Paths: []string{"../module-h"}}, + Terraform: &config.TerraformConfig{Source: ptr("fire")}, + IsPartial: true, + GenerateConfigs: make(map[string]codegen.GenerateConfig), + }, + TerragruntOptions: mockOptions.Clone(canonical(t, "../test/fixture-modules/module-k/"+config.DefaultTerragruntConfigPath)), + } + + configPaths := []string{"../test/fixture-modules/module-j/" + config.DefaultTerragruntConfigPath, "../test/fixture-modules/module-k/" + config.DefaultTerragruntConfigPath} + expected := TerraformModules{moduleH, moduleI, moduleJ, moduleK} + + stack := NewStack(mockOptions) + actualModules, actualErr := stack.ResolveTerraformModules(context.Background(), configPaths) + require.NoError(t, actualErr) + assertModuleListsEqual(t, expected, actualModules) +} + +func TestResolveTerraformModulesInvalidPaths(t *testing.T) { + t.Parallel() + + configPaths := []string{"../test/fixture-modules/module-missing-dependency/" + config.DefaultTerragruntConfigPath} + + stack := NewStack(mockOptions) + _, actualErr := stack.ResolveTerraformModules(context.Background(), configPaths) + require.Error(t, actualErr) + + underlying, ok := errors.Unwrap(actualErr).(ProcessingModuleError) + require.True(t, ok) + + unwrapped := errors.Unwrap(underlying.UnderlyingError) + assert.True(t, os.IsNotExist(unwrapped), "Expected a file not exists error but got %v", underlying.UnderlyingError) +} + +func TestResolveTerraformModuleNoTerraformConfig(t *testing.T) { + t.Parallel() + + configPaths := []string{"../test/fixture-modules/module-l/" + config.DefaultTerragruntConfigPath} + expected := TerraformModules{} + + stack := NewStack(mockOptions) + actualModules, actualErr := stack.ResolveTerraformModules(context.Background(), configPaths) + assert.Nil(t, actualErr, "Unexpected error: %v", actualErr) + assertModuleListsEqual(t, expected, actualModules) +} + +func TestBasicDependency(t *testing.T) { + moduleC := &TerraformModule{Path: "C", Dependencies: TerraformModules{}} + moduleB := &TerraformModule{Path: "B", Dependencies: TerraformModules{moduleC}} + moduleA := &TerraformModule{Path: "A", Dependencies: TerraformModules{moduleB}} + + stack := NewStack(&options.TerragruntOptions{WorkingDir: "test-stack"}) + stack.Modules = TerraformModules{moduleA, moduleB, moduleC} + + expected := map[string][]string{ + "B": {"A"}, + "C": {"B", "A"}, + } + + result := stack.ListStackDependentModules() + + if !reflect.DeepEqual(result, expected) { + t.Errorf("Expected %v, got %v", expected, result) + } +} +func TestNestedDependencies(t *testing.T) { + moduleD := &TerraformModule{Path: "D", Dependencies: TerraformModules{}} + moduleC := &TerraformModule{Path: "C", Dependencies: TerraformModules{moduleD}} + moduleB := &TerraformModule{Path: "B", Dependencies: TerraformModules{moduleC}} + moduleA := &TerraformModule{Path: "A", Dependencies: TerraformModules{moduleB}} + + // Create a mock stack + stack := NewStack(&options.TerragruntOptions{WorkingDir: "nested-stack"}) + stack.Modules = TerraformModules{moduleA, moduleB, moduleC, moduleD} + + // Expected result + expected := map[string][]string{ + "B": {"A"}, + "C": {"B", "A"}, + "D": {"C", "B", "A"}, + } + + // Run the function + result := stack.ListStackDependentModules() + + if !reflect.DeepEqual(result, expected) { + t.Errorf("Expected %v, got %v", expected, result) + } +} + +func TestCircularDependencies(t *testing.T) { + // Mock modules with circular dependencies + moduleA := &TerraformModule{Path: "A"} + moduleB := &TerraformModule{Path: "B"} + moduleC := &TerraformModule{Path: "C"} + + moduleA.Dependencies = TerraformModules{moduleB} + moduleB.Dependencies = TerraformModules{moduleC} + moduleC.Dependencies = TerraformModules{moduleA} // Circular dependency + + stack := NewStack(&options.TerragruntOptions{WorkingDir: "circular-stack"}) + stack.Modules = TerraformModules{moduleA, moduleB, moduleC} + + expected := map[string][]string{ + "A": {"C", "B"}, + "B": {"A", "C"}, + "C": {"B", "A"}, + } + + // Run the function + result := stack.ListStackDependentModules() + + if !reflect.DeepEqual(result, expected) { + t.Errorf("Expected %v, got %v", expected, result) + } +} diff --git a/configstack/test_helpers.go b/configstack/test_helpers.go index c38ed0e41..8df3d415f 100644 --- a/configstack/test_helpers.go +++ b/configstack/test_helpers.go @@ -13,7 +13,7 @@ import ( "github.com/stretchr/testify/assert" ) -type TerraformModuleByPath []*TerraformModule +type TerraformModuleByPath TerraformModules func (byPath TerraformModuleByPath) Len() int { return len(byPath) } func (byPath TerraformModuleByPath) Swap(i, j int) { byPath[i], byPath[j] = byPath[j], byPath[i] } @@ -29,7 +29,7 @@ func (byPath RunningModuleByPath) Less(i, j int) bool { // We can't use assert.Equals on TerraformModule or any data structure that contains it because it contains some // fields (e.g. TerragruntOptions) that cannot be compared directly -func assertModuleListsEqual(t *testing.T, expectedModules []*TerraformModule, actualModules []*TerraformModule, messageAndArgs ...interface{}) { +func assertModuleListsEqual(t *testing.T, expectedModules TerraformModules, actualModules TerraformModules, messageAndArgs ...interface{}) { if !assert.Equal(t, len(expectedModules), len(actualModules), messageAndArgs...) { t.Logf("%s != %s", expectedModules, actualModules) return @@ -120,13 +120,13 @@ func assertRunningModulesEqual(t *testing.T, expected *runningModule, actual *ru } } -// We can't do a simple IsError comparison for UnrecognizedDependency because that error is a struct that +// We can't do a simple IsError comparison for UnrecognizedDependencyError because that error is a struct that // contains an array, and in Go, trying to compare arrays gives a "comparing uncomparable type -// configstack.UnrecognizedDependency" panic. Therefore, we have to compare that error more manually. +// configstack.UnrecognizedDependencyError" panic. Therefore, we have to compare that error more manually. func assertErrorsEqual(t *testing.T, expected error, actual error, messageAndArgs ...interface{}) { actual = errors.Unwrap(actual) - if expectedUnrecognized, isUnrecognizedDependencyError := expected.(UnrecognizedDependency); isUnrecognizedDependencyError { - actualUnrecognized, isUnrecognizedDependencyError := actual.(UnrecognizedDependency) + if expectedUnrecognized, isUnrecognizedDependencyError := expected.(UnrecognizedDependencyError); isUnrecognizedDependencyError { + actualUnrecognized, isUnrecognizedDependencyError := actual.(UnrecognizedDependencyError) if assert.True(t, isUnrecognizedDependencyError, messageAndArgs...) { assert.Equal(t, expectedUnrecognized, actualUnrecognized, messageAndArgs...) } diff --git a/docs/_docs/04_reference/cli-options.md b/docs/_docs/04_reference/cli-options.md index e36939bbf..b25d3c710 100644 --- a/docs/_docs/04_reference/cli-options.md +++ b/docs/_docs/04_reference/cli-options.md @@ -25,6 +25,7 @@ This page documents the CLI commands and options available with Terragrunt: - [validate-inputs](#validate-inputs) - [graph-dependencies](#graph-dependencies) - [hclfmt](#hclfmt) + - [hclvalidate](#hclvalidate) - [aws-provider-patch](#aws-provider-patch) - [render-json](#render-json) - [output-module-groups](#output-module-groups) @@ -62,6 +63,8 @@ This page documents the CLI commands and options available with Terragrunt: - [terragrunt-check](#terragrunt-check) - [terragrunt-diff](#terragrunt-diff) - [terragrunt-hclfmt-file](#terragrunt-hclfmt-file) + - [terragrunt-hclvalidate-json](#terragrunt-hclvalidate-json) + - [terragrunt-hclvalidate-invalid](#terragrunt-hclvalidate-invalid) - [terragrunt-override-attr](#terragrunt-override-attr) - [terragrunt-json-out](#terragrunt-json-out) - [terragrunt-json-disable-dependent-modules](#terragrunt-json-disable-dependent-modules) @@ -98,6 +101,7 @@ Terragrunt supports the following CLI commands: - [validate-inputs](#validate-inputs) - [graph-dependencies](#graph-dependencies) - [hclfmt](#hclfmt) +- [hclvalidate](#hclvalidate) - [aws-provider-patch](#aws-provider-patch) - [render-json](#render-json) - [output-module-groups](#output-module-groups) @@ -407,6 +411,19 @@ terragrunt hclfmt This will recursively search the current working directory for any folders that contain Terragrunt configuration files and run the equivalent of `terraform fmt` on them. +### hclvalidate + +Find all hcl files from the configuration stack and validate them. + +Example: + +```bash +terragrunt hclvalidate +``` + +This will search all hcl files from the configuration stack in the current working directory and run the equivalent +of `terraform validate` on them. + ### aws-provider-patch Overwrite settings on nested AWS providers to work around several Terraform bugs. Due to @@ -730,6 +747,8 @@ prefix `--terragrunt-` (e.g., `--terragrunt-config`). The currently available op - [terragrunt-check](#terragrunt-check) - [terragrunt-diff](#terragrunt-diff) - [terragrunt-hclfmt-file](#terragrunt-hclfmt-file) + - [terragrunt-hclvalidate-json](#terragrunt-hclvalidate-json) + - [terragrunt-hclvalidate-invalid](#terragrunt-hclvalidate-invalid) - [terragrunt-override-attr](#terragrunt-override-attr) - [terragrunt-json-out](#terragrunt-json-out) - [terragrunt-json-disable-dependent-modules](#terragrunt-json-disable-dependent-modules) @@ -1079,6 +1098,26 @@ When passed in, running `hclfmt` will print diff between original and modified f When passed in, run `hclfmt` only on specified hcl file. +### terragrunt-hclvalidate-json + +**CLI Arg**: `--terragrunt-hclvalidate-json`
+**Environment Variable**: `TERRAGRUNT_HCLVALIDATE_JSON` (set to `true`)
+**Commands**: + +- [hclvalidate](#hclvalidate) + +When passed in, render the output in the JSON format. + +### terragrunt-hclvalidate-invalid + +**CLI Arg**: `--terragrunt-hclvalidate-invalid`
+**Environment Variable**: `TERRAGRUNT_HCLVALIDATE_INVALID` (set to `true`)
+**Commands**: + +- [hclvalidate](#hclvalidate) + +When passed in, output a list of files with invalid configuration. + ### terragrunt-override-attr **CLI Arg**: `--terragrunt-override-attr`
diff --git a/go.mod b/go.mod index f2f9cb8eb..4b6db1d52 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/fatih/structs v1.1.0 github.com/go-errors/errors v1.4.2 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 - github.com/gruntwork-io/terratest v0.41.0 + github.com/gruntwork-io/terratest v0.46.16 github.com/hashicorp/go-cleanhttp v0.5.2 github.com/hashicorp/go-getter v1.7.5 github.com/hashicorp/go-multierror v1.1.1 @@ -42,7 +42,7 @@ require ( github.com/hashicorp/hcl v1.0.1-vault // indirect github.com/hashicorp/vault/api v1.10.0 // indirect github.com/klauspost/compress v1.17.7 // indirect - github.com/mitchellh/go-wordwrap v1.0.1 // indirect + github.com/mitchellh/go-wordwrap v1.0.1 github.com/opencontainers/runc v1.1.12 // indirect github.com/pquerna/otp v1.2.1-0.20191009055518-468c2dd2b58d // indirect github.com/terraform-linters/tflint v0.47.0 @@ -63,10 +63,11 @@ require ( github.com/gofrs/flock v0.8.1 github.com/google/uuid v1.6.0 github.com/gruntwork-io/boilerplate v0.5.11 - github.com/gruntwork-io/go-commons v0.17.1 + github.com/gruntwork-io/go-commons v0.17.2 github.com/gruntwork-io/gruntwork-cli v0.7.0 github.com/hashicorp/go-getter/v2 v2.2.1 github.com/labstack/echo/v4 v4.11.4 + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c github.com/pkg/errors v0.9.1 github.com/posener/complete v1.2.3 @@ -143,7 +144,6 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/dlclark/regexp2 v1.4.0 // indirect github.com/docker/cli v24.0.0+incompatible // indirect - github.com/docker/docker v24.0.9+incompatible // indirect github.com/docker/go-units v0.5.0 // indirect github.com/fatih/color v1.16.0 // indirect github.com/getsops/gopgagent v0.0.0-20170926210634-4d7ea76ff71a // indirect @@ -243,7 +243,7 @@ require ( google.golang.org/protobuf v1.33.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - sigs.k8s.io/yaml v1.2.0 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect ) // This is necessary to workaround go modules error with terraform importing vault incorrectly. diff --git a/go.sum b/go.sum index 52d6bb648..dfe979978 100644 --- a/go.sum +++ b/go.sum @@ -642,12 +642,12 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 h1:Wqo399gCIufwto+VfwCSvsnfGpF github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0/go.mod h1:qmOFXW2epJhM0qSnUUYpldc7gVz2KMQwJ/QYCDIa7XU= github.com/gruntwork-io/boilerplate v0.5.11 h1:ifsrOyvKidF+/3Mn9au8hF5lWQo/hG+gk3Ie5ahMufA= github.com/gruntwork-io/boilerplate v0.5.11/go.mod h1:A6sNcRrNICYAMwqP6fr1n/k6/u1VRORUkqACEhfHrYs= -github.com/gruntwork-io/go-commons v0.17.1 h1:2KS9wAqrgeOTWj33DSHzDNJ1FCprptWdLFqej+wB8x0= -github.com/gruntwork-io/go-commons v0.17.1/go.mod h1:S98JcR7irPD1bcruSvnqupg+WSJEJ6xaM89fpUZVISk= +github.com/gruntwork-io/go-commons v0.17.2 h1:14dsCJ7M5Vv2X3BIPKeG9Kdy6vTMGhM8L4WZazxfTuY= +github.com/gruntwork-io/go-commons v0.17.2/go.mod h1:zs7Q2AbUKuTarBPy19CIxJVUX/rBamfW8IwuWKniWkE= github.com/gruntwork-io/gruntwork-cli v0.7.0 h1:YgSAmfCj9c61H+zuvHwKfYUwlMhu5arnQQLM4RH+CYs= github.com/gruntwork-io/gruntwork-cli v0.7.0/go.mod h1:jp6Z7NcLF2avpY8v71fBx6hds9eOFPELSuD/VPv7w00= -github.com/gruntwork-io/terratest v0.41.0 h1:QKFK6m0EMVnrV7lw2L06TlG+Ha3t0CcOXuBVywpeNRU= -github.com/gruntwork-io/terratest v0.41.0/go.mod h1:qH1xkPTTGx30XkMHw8jAVIbzqheSjIa5IyiTwSV2vKI= +github.com/gruntwork-io/terratest v0.46.16 h1:l+HHuU7lNLwoAl2sP8zkYJy0uoE2Mwha2nw+rim+OhQ= +github.com/gruntwork-io/terratest v0.46.16/go.mod h1:oywHw1cFKXSYvKPm27U7quZVzDUlA22H2xUrKCe26xM= github.com/hashicorp/aws-sdk-go-base v0.6.0/go.mod h1:2fRjWDv3jJBeN6mVWFHV6hFTNeFBx2gpDLQaZNxUVAY= github.com/hashicorp/consul v0.0.0-20171026175957-610f3c86a089/go.mod h1:mFrjN1mfidgJfYP1xrJCF+AfRhr6Eaqhb2+sfyn/OOI= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -854,6 +854,7 @@ github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXm github.com/miekg/dns v1.0.8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.1.2/go.mod h1:6iaV0fGdElS6dPBx0EApTxHrcWvmJphyh2n8YBLPPZ4= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= @@ -979,8 +980,9 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -1792,5 +1794,5 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/internal/view/diagnostic/diagnostic.go b/internal/view/diagnostic/diagnostic.go new file mode 100644 index 000000000..e25158021 --- /dev/null +++ b/internal/view/diagnostic/diagnostic.go @@ -0,0 +1,58 @@ +package diagnostic + +import ( + "github.com/hashicorp/hcl/v2" +) + +type Diagnostics []*Diagnostic + +func (diags *Diagnostics) Contains(find *hcl.Diagnostic) bool { + for _, diag := range *diags { + if find.Subject != nil && find.Subject.String() == diag.Range.String() { + return true + } + } + return false +} + +type Diagnostic struct { + Severity DiagnosticSeverity `json:"severity"` + Summary string `json:"summary"` + Detail string `json:"detail"` + Range *Range `json:"range,omitempty"` + Snippet *Snippet `json:"snippet,omitempty"` +} + +func NewDiagnostic(file *hcl.File, hclDiag *hcl.Diagnostic) *Diagnostic { + diag := &Diagnostic{ + Severity: DiagnosticSeverity(hclDiag.Severity), + Summary: hclDiag.Summary, + Detail: hclDiag.Detail, + } + + if hclDiag.Subject == nil { + return diag + } + + highlightRange := *hclDiag.Subject + if highlightRange.Empty() { + highlightRange.End.Byte++ + highlightRange.End.Column++ + } + diag.Snippet = NewSnippet(file, hclDiag, highlightRange) + + diag.Range = &Range{ + Filename: highlightRange.Filename, + Start: Pos{ + Line: highlightRange.Start.Line, + Column: highlightRange.Start.Column, + Byte: highlightRange.Start.Byte, + }, + End: Pos{ + Line: highlightRange.End.Line, + Column: highlightRange.End.Column, + Byte: highlightRange.End.Byte, + }, + } + return diag +} diff --git a/internal/view/diagnostic/expression_value.go b/internal/view/diagnostic/expression_value.go new file mode 100644 index 000000000..099628549 --- /dev/null +++ b/internal/view/diagnostic/expression_value.go @@ -0,0 +1,164 @@ +package diagnostic + +import ( + "bytes" + "fmt" + "sort" + + "github.com/hashicorp/hcl/v2" + "github.com/zclconf/go-cty/cty" +) + +const ( + // Sensitive indicates that this value is marked as sensitive in the context of Terraform. + Sensitive = valueMark("Sensitive") +) + +// valueMarks allow creating strictly typed values for use as cty.Value marks. +type valueMark string + +func (m valueMark) GoString() string { + return "marks." + string(m) +} + +// ExpressionValue represents an HCL traversal string and a statement about its value while the expression was evaluated. +type ExpressionValue struct { + Traversal string `json:"traversal"` + Statement string `json:"statement"` +} + +func DescribeExpressionValues(hclDiag *hcl.Diagnostic) []ExpressionValue { + var ( + expr = hclDiag.Expression + ctx = hclDiag.EvalContext + + vars = expr.Variables() + values = make([]ExpressionValue, 0, len(vars)) + seen = make(map[string]struct{}, len(vars)) + includeUnknown = DiagnosticCausedByUnknown(hclDiag) + includeSensitive = DiagnosticCausedBySensitive(hclDiag) + ) + +Traversals: + for _, traversal := range vars { + for len(traversal) > 1 { + val, diags := traversal.TraverseAbs(ctx) + if diags.HasErrors() { + traversal = traversal[:len(traversal)-1] + continue + } + + traversalStr := traversalStr(traversal) + if _, exists := seen[traversalStr]; exists { + continue Traversals + } + value := ExpressionValue{ + Traversal: traversalStr, + } + switch { + + case val.HasMark(Sensitive): + if !includeSensitive { + continue Traversals + } + value.Statement = "has a sensitive value" + case !val.IsKnown(): + if ty := val.Type(); ty != cty.DynamicPseudoType { + if includeUnknown { + value.Statement = fmt.Sprintf("is a %s, known only after apply", ty.FriendlyName()) + } else { + value.Statement = fmt.Sprintf("is a %s", ty.FriendlyName()) + } + } else { + if !includeUnknown { + continue Traversals + } + value.Statement = "will be known only after apply" + } + default: + value.Statement = fmt.Sprintf("is %s", valueStr(val)) + } + values = append(values, value) + seen[traversalStr] = struct{}{} + } + } + sort.Slice(values, func(i, j int) bool { + return values[i].Traversal < values[j].Traversal + }) + + return values +} + +func traversalStr(traversal hcl.Traversal) string { + var buf bytes.Buffer + for _, step := range traversal { + switch tStep := step.(type) { + case hcl.TraverseRoot: + buf.WriteString(tStep.Name) + case hcl.TraverseAttr: + buf.WriteByte('.') + buf.WriteString(tStep.Name) + case hcl.TraverseIndex: + buf.WriteByte('[') + if keyTy := tStep.Key.Type(); keyTy.IsPrimitiveType() { + buf.WriteString(valueStr(tStep.Key)) + } else { + // We'll just use a placeholder for more complex values, since otherwise our result could grow ridiculously long. + buf.WriteString("...") + } + buf.WriteByte(']') + } + } + return buf.String() +} + +func valueStr(val cty.Value) string { + if val.HasMark(Sensitive) { + return "(sensitive value)" + } + ty := val.Type() + switch { + case val.IsNull(): + return "null" + case !val.IsKnown(): + return "(not yet known)" + case ty == cty.Bool: + if val.True() { + return "true" + } + return "false" + case ty == cty.Number: + bf := val.AsBigFloat() + prec := 10 + return bf.Text('g', prec) + case ty == cty.String: + return fmt.Sprintf("%q", val.AsString()) + case ty.IsCollectionType() || ty.IsTupleType(): + l := val.LengthInt() + switch l { + case 0: + return "empty " + ty.FriendlyName() + case 1: + return ty.FriendlyName() + " with 1 element" + default: + return fmt.Sprintf("%s with %d elements", ty.FriendlyName(), l) + } + case ty.IsObjectType(): + atys := ty.AttributeTypes() + l := len(atys) + switch l { + case 0: + return "object with no attributes" + case 1: + var name string + for k := range atys { + name = k + } + return fmt.Sprintf("object with 1 attribute %q", name) + default: + return fmt.Sprintf("object with %d attributes", l) + } + default: + return ty.FriendlyName() + } +} diff --git a/internal/view/diagnostic/extra.go b/internal/view/diagnostic/extra.go new file mode 100644 index 000000000..166b2c8b3 --- /dev/null +++ b/internal/view/diagnostic/extra.go @@ -0,0 +1,73 @@ +package diagnostic + +import "github.com/hashicorp/hcl/v2" + +func ExtraInfo[T any](diag *hcl.Diagnostic) T { + extra := diag.Extra + if ret, ok := extra.(T); ok { + return ret + } + + // If "extra" doesn't implement T directly then we'll delegate to our ExtraInfoNext helper to try iteratively unwrapping it. + return ExtraInfoNext[T](extra) +} + +// ExtraInfoNext takes a value previously returned by ExtraInfo and attempts to find an implementation of interface T wrapped inside of it. The return value meaning is the same as for ExtraInfo. +func ExtraInfoNext[T any](previous interface{}) T { + // As long as T is an interface type as documented, zero will always be a nil interface value for us to return in the non-matching case. + var zero T + + unwrapper, ok := previous.(DiagnosticExtraUnwrapper) + // If the given value isn't unwrappable then it can't possibly have any other info nested inside of it. + if !ok { + return zero + } + + extra := unwrapper.UnwrapDiagnosticExtra() + + // Keep unwrapping until we either find the interface to look for or we run out of layers of unwrapper. + for { + if ret, ok := extra.(T); ok { + return ret + } + + if unwrapper, ok := extra.(DiagnosticExtraUnwrapper); ok { + extra = unwrapper.UnwrapDiagnosticExtra() + } else { + return zero + } + } +} + +// DiagnosticExtraUnwrapper is an interface implemented by values in the Extra field of Diagnostic when they are wrapping another "Extra" value that was generated downstream. +type DiagnosticExtraUnwrapper interface { + UnwrapDiagnosticExtra() interface{} +} + +// DiagnosticExtraBecauseUnknown is an interface implemented by values in the Extra field of Diagnostic when the diagnostic is potentially caused by the presence of unknown values in an expression evaluation. +type DiagnosticExtraBecauseUnknown interface { + DiagnosticCausedByUnknown() bool +} + +// DiagnosticCausedByUnknown returns true if the given diagnostic has an indication that it was caused by the presence of unknown values during an expression evaluation. +func DiagnosticCausedByUnknown(diag *hcl.Diagnostic) bool { + maybe := ExtraInfo[DiagnosticExtraBecauseUnknown](diag) + if maybe == nil { + return false + } + return maybe.DiagnosticCausedByUnknown() +} + +// DiagnosticExtraBecauseSensitive is an interface implemented by values in the Extra field of Diagnostic when the diagnostic is potentially caused by the presence of sensitive values in an expression evaluation. +type DiagnosticExtraBecauseSensitive interface { + DiagnosticCausedBySensitive() bool +} + +// DiagnosticCausedBySensitive returns true if the given diagnostic has an/ indication that it was caused by the presence of sensitive values during an expression evaluation. +func DiagnosticCausedBySensitive(diag *hcl.Diagnostic) bool { + maybe := ExtraInfo[DiagnosticExtraBecauseSensitive](diag) + if maybe == nil { + return false + } + return maybe.DiagnosticCausedBySensitive() +} diff --git a/internal/view/diagnostic/function.go b/internal/view/diagnostic/function.go new file mode 100644 index 000000000..c0e62fead --- /dev/null +++ b/internal/view/diagnostic/function.go @@ -0,0 +1,120 @@ +package diagnostic + +import ( + "encoding/json" + "strings" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/function" +) + +// FunctionParam represents a single parameter to a function, as represented by type Function. +type FunctionParam struct { + // Name is a name for the function which is used primarily for documentation purposes. + Name string `json:"name"` + + // Type is a type constraint which is a static approximation of the possibly-dynamic type of the parameter + Type json.RawMessage `json:"type"` + + Description string `json:"description,omitempty"` + DescriptionKind string `json:"description_kind,omitempty"` +} + +func DescribeFunctionParam(p *function.Parameter) FunctionParam { + ret := FunctionParam{ + Name: p.Name, + } + + if raw, err := p.Type.MarshalJSON(); err != nil { + // Treat any errors as if the function is dynamically typed because it would be weird to get here. + ret.Type = json.RawMessage(`"dynamic"`) + } else { + ret.Type = raw + } + + return ret +} + +// Function is a description of the JSON representation of the signature of a function callable from the Terraform language. +type Function struct { + // Name is the leaf name of the function, without any namespace prefix. + Name string `json:"name"` + + Params []FunctionParam `json:"params"` + VariadicParam *FunctionParam `json:"variadic_param,omitempty"` + + // ReturnType is type constraint which is a static approximation of the possibly-dynamic return type of the function. + ReturnType json.RawMessage `json:"return_type"` + + Description string `json:"description,omitempty"` + DescriptionKind string `json:"description_kind,omitempty"` +} + +// DescribeFunction returns a description of the signature of the given cty function, as a pointer to this package's serializable type Function. +func DescribeFunction(name string, f function.Function) *Function { + ret := &Function{ + Name: name, + } + + params := f.Params() + ret.Params = make([]FunctionParam, len(params)) + typeCheckArgs := make([]cty.Type, len(params), len(params)+1) + for i, param := range params { + ret.Params[i] = DescribeFunctionParam(¶m) + typeCheckArgs[i] = param.Type + } + if varParam := f.VarParam(); varParam != nil { + descParam := DescribeFunctionParam(varParam) + ret.VariadicParam = &descParam + typeCheckArgs = append(typeCheckArgs, varParam.Type) + } + + retType, err := f.ReturnType(typeCheckArgs) + if err != nil { + retType = cty.DynamicPseudoType + } + + if raw, err := retType.MarshalJSON(); err != nil { + // Treat any errors as if the function is dynamically typed because it would be weird to get here. + ret.ReturnType = json.RawMessage(`"dynamic"`) + } else { + ret.ReturnType = raw + } + + return ret +} + +// FunctionCall represents a function call whose information is being included as part of a diagnostic snippet. +type FunctionCall struct { + // CalledAs is the full name that was used to call this function, potentially including namespace prefixes if the function does not belong to the default function namespace. + CalledAs string `json:"called_as"` + + // Signature is a description of the signature of the function that was/ called, if any. + Signature *Function `json:"signature,omitempty"` +} + +func DescribeFunctionCall(hclDiag *hcl.Diagnostic) *FunctionCall { + callInfo := ExtraInfo[hclsyntax.FunctionCallDiagExtra](hclDiag) + if callInfo == nil || callInfo.CalledFunctionName() == "" { + return nil + } + + calledAs := callInfo.CalledFunctionName() + baseName := calledAs + if idx := strings.LastIndex(baseName, "::"); idx >= 0 { + baseName = baseName[idx+2:] + } + + var signature *Function + + if f, ok := hclDiag.EvalContext.Functions[calledAs]; ok { + signature = DescribeFunction(baseName, f) + } + + return &FunctionCall{ + CalledAs: calledAs, + Signature: signature, + } +} diff --git a/internal/view/diagnostic/range.go b/internal/view/diagnostic/range.go new file mode 100644 index 000000000..b0f01ad5f --- /dev/null +++ b/internal/view/diagnostic/range.go @@ -0,0 +1,40 @@ +package diagnostic + +import "fmt" + +// Pos represents a position in the source code. +type Pos struct { + // Line is a one-based count for the line in the indicated file. + Line int `json:"line"` + + // Column is a one-based count of Unicode characters from the start of the line. + Column int `json:"column"` + + // Byte is a zero-based offset into the indicated file. + Byte int `json:"byte"` +} + +// Range represents the filename and position of the diagnostic subject. +type Range struct { + Filename string `json:"filename"` + Start Pos `json:"start"` + End Pos `json:"end"` +} + +func (rng Range) String() string { + if rng.Start.Line == rng.End.Line { + return fmt.Sprintf( + "%s:%d,%d-%d", + rng.Filename, + rng.Start.Line, rng.Start.Column, + rng.End.Column, + ) + } else { + return fmt.Sprintf( + "%s:%d,%d-%d,%d", + rng.Filename, + rng.Start.Line, rng.Start.Column, + rng.End.Line, rng.End.Column, + ) + } +} diff --git a/internal/view/diagnostic/servity.go b/internal/view/diagnostic/servity.go new file mode 100644 index 000000000..089618f08 --- /dev/null +++ b/internal/view/diagnostic/servity.go @@ -0,0 +1,41 @@ +package diagnostic + +import ( + "fmt" + "strings" + + "github.com/hashicorp/hcl/v2" +) + +const ( + DiagnosticSeverityUnknown = "unknown" + DiagnosticSeverityError = "error" + DiagnosticSeverityWarning = "warning" +) + +type DiagnosticSeverity hcl.DiagnosticSeverity + +func (severity DiagnosticSeverity) String() string { + switch hcl.DiagnosticSeverity(severity) { + case hcl.DiagError: + return DiagnosticSeverityError + case hcl.DiagWarning: + return DiagnosticSeverityWarning + default: + return DiagnosticSeverityUnknown + } +} + +func (severity DiagnosticSeverity) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf(`"%s"`, severity.String())), nil +} + +func (severity *DiagnosticSeverity) UnmarshalJSON(val []byte) error { + switch strings.Trim(string(val), `"`) { + case DiagnosticSeverityError: + *severity = DiagnosticSeverity(hcl.DiagError) + case DiagnosticSeverityWarning: + *severity = DiagnosticSeverity(hcl.DiagWarning) + } + return nil +} diff --git a/internal/view/diagnostic/snippet.go b/internal/view/diagnostic/snippet.go new file mode 100644 index 000000000..268913504 --- /dev/null +++ b/internal/view/diagnostic/snippet.go @@ -0,0 +1,95 @@ +package diagnostic + +import ( + "bufio" + "strings" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hcled" +) + +// Snippet represents source code information about the diagnostic. +type Snippet struct { + // Context is derived from HCL's hcled.ContextString output. This gives a high-level summary of the root context of the diagnostic. + Context string `json:"context"` + + // Code is a possibly-multi-line string of Terraform configuration, which includes both the diagnostic source and any relevant context as defined by the diagnostic. + Code string `json:"code"` + + // StartLine is the line number in the source file for the first line of the snippet code block. + StartLine int `json:"start_line"` + + // HighlightStartOffset is the character offset into Code at which the diagnostic source range starts, which ought to be highlighted as such by the consumer of this data. + HighlightStartOffset int `json:"highlight_start_offset"` + + // HighlightEndOffset is the character offset into Code at which the diagnostic source range ends. + HighlightEndOffset int `json:"highlight_end_offset"` + + // Values is a sorted slice of expression values which may be useful in understanding the source of an error in a complex expression. + Values []ExpressionValue `json:"values"` + + // FunctionCall is information about a function call whose failure is being reported by this diagnostic, if any. + FunctionCall *FunctionCall `json:"function_call,omitempty"` +} + +func NewSnippet(file *hcl.File, hclDiag *hcl.Diagnostic, highlightRange hcl.Range) *Snippet { + snipRange := *hclDiag.Subject + if hclDiag.Context != nil { + // Show enough of the source code to include both the subject and context ranges, which overlap in all reasonable situations. + snipRange = hcl.RangeOver(snipRange, *hclDiag.Context) + } + + if snipRange.Empty() { + snipRange.End.Byte++ + snipRange.End.Column++ + } + + snippet := &Snippet{ + StartLine: hclDiag.Subject.Start.Line, + } + + if file != nil && file.Bytes != nil { + snippet.Context = hcled.ContextString(file, hclDiag.Subject.Start.Byte-1) + + var codeStartByte int + sc := hcl.NewRangeScanner(file.Bytes, hclDiag.Subject.Filename, bufio.ScanLines) + var code strings.Builder + for sc.Scan() { + lineRange := sc.Range() + if lineRange.Overlaps(snipRange) { + if codeStartByte == 0 && code.Len() == 0 { + codeStartByte = lineRange.Start.Byte + } + code.Write(lineRange.SliceBytes(file.Bytes)) + code.WriteRune('\n') + } + } + codeStr := strings.TrimSuffix(code.String(), "\n") + snippet.Code = codeStr + + start := highlightRange.Start.Byte - codeStartByte + end := start + (highlightRange.End.Byte - highlightRange.Start.Byte) + + if start < 0 { + start = 0 + } else if start > len(codeStr) { + start = len(codeStr) + } + if end < 0 { + end = 0 + } else if end > len(codeStr) { + end = len(codeStr) + } + + snippet.HighlightStartOffset = start + snippet.HighlightEndOffset = end + } + + if hclDiag.Expression == nil || hclDiag.EvalContext == nil { + return snippet + } + + snippet.Values = DescribeExpressionValues(hclDiag) + snippet.FunctionCall = DescribeFunctionCall(hclDiag) + return snippet +} diff --git a/internal/view/human_render.go b/internal/view/human_render.go new file mode 100644 index 000000000..fc0167397 --- /dev/null +++ b/internal/view/human_render.go @@ -0,0 +1,271 @@ +package view + +import ( + "bufio" + "bytes" + "fmt" + "os" + "sort" + "strings" + + "github.com/gruntwork-io/go-commons/errors" + "github.com/gruntwork-io/terragrunt/internal/view/diagnostic" + "github.com/hashicorp/hcl/v2" + "github.com/mitchellh/colorstring" + "github.com/mitchellh/go-wordwrap" + "golang.org/x/term" +) + +const defaultWidth = 78 + +type HumanRender struct { + colorize *colorstring.Colorize + width int +} + +func NewHumanRender(disableColor bool) Render { + disableColor = disableColor || !term.IsTerminal(int(os.Stderr.Fd())) + width, _, err := term.GetSize(int(os.Stdout.Fd())) + if err != nil { + width = defaultWidth + } + + return &HumanRender{ + colorize: &colorstring.Colorize{ + Colors: colorstring.DefaultColors, + Disable: disableColor, + Reset: true, + }, + width: width, + } +} + +func (render *HumanRender) InvalidConfigPath(filenames []string) (string, error) { + var buf bytes.Buffer + + for _, filename := range filenames { + buf.WriteString(filename) + buf.WriteByte('\n') + } + + return buf.String(), nil +} + +func (render *HumanRender) Diagnostics(diags diagnostic.Diagnostics) (string, error) { + var buf bytes.Buffer + + for _, diag := range diags { + str, err := render.Diagnostic(diag) + if err != nil { + return "", err + } + if str != "" { + buf.WriteString(str) + buf.WriteByte('\n') + } + } + + return buf.String(), nil +} + +// Diagnostic formats a single diagnostic message. +func (render *HumanRender) Diagnostic(diag *diagnostic.Diagnostic) (string, error) { + var buf bytes.Buffer + + // these leftRule* variables are markers for the beginning of the lines + // containing the diagnostic that are intended to help sighted users + // better understand the information hierarchy when diagnostics appear + // alongside other information or alongside other diagnostics. + // + // Without this, it seems (based on folks sharing incomplete messages when + // asking questions, or including extra content that's not part of the + // diagnostic) that some readers have trouble easily identifying which + // text belongs to the diagnostic and which does not. + var leftRuleLine, leftRuleStart, leftRuleEnd string + var leftRuleWidth int // in visual character cells + + switch hcl.DiagnosticSeverity(diag.Severity) { + case hcl.DiagError: + buf.WriteString(render.colorize.Color("[bold][red]Error: [reset]")) + leftRuleLine = render.colorize.Color("[red]│[reset] ") + leftRuleStart = render.colorize.Color("[red]╷[reset]") + leftRuleEnd = render.colorize.Color("[red]╵[reset]") + leftRuleWidth = 2 + case hcl.DiagWarning: + buf.WriteString(render.colorize.Color("[bold][yellow]Warning: [reset]")) + leftRuleLine = render.colorize.Color("[yellow]│[reset] ") + leftRuleStart = render.colorize.Color("[yellow]╷[reset]") + leftRuleEnd = render.colorize.Color("[yellow]╵[reset]") + leftRuleWidth = 2 + default: + // Clear out any coloring that might be applied by Terraform's UI helper, + // so our result is not context-sensitive. + buf.WriteString(render.colorize.Color("\n[reset]")) + } + + // We don't wrap the summary, since we expect it to be terse, and since + // this is where we put the text of a native Go error it may not always + // be pure text that lends itself well to word-wrapping. + if _, err := fmt.Fprintf(&buf, render.colorize.Color("[bold]%s[reset]\n\n"), diag.Summary); err != nil { + return "", errors.WithStackTrace(err) + } + + sourceSnippets, err := render.SourceSnippets(diag) + if err != nil { + return "", err + } + buf.WriteString(sourceSnippets) + + if diag.Detail != "" { + paraWidth := render.width - leftRuleWidth - 1 // leave room for the left rule + if paraWidth > 0 { + lines := strings.Split(diag.Detail, "\n") + for _, line := range lines { + if !strings.HasPrefix(line, " ") { + line = wordwrap.WrapString(line, uint(paraWidth)) + } + if _, err := fmt.Fprintf(&buf, "%s\n", line); err != nil { + return "", errors.WithStackTrace(err) + } + } + } else { + if _, err := fmt.Fprintf(&buf, "%s\n", diag.Detail); err != nil { + return "", errors.WithStackTrace(err) + } + } + } + + // Before we return, we'll finally add the left rule prefixes to each + // line so that the overall message is visually delimited from what's + // around it. We'll do that by scanning over what we already generated + // and adding the prefix for each line. + var ruleBuf strings.Builder + sc := bufio.NewScanner(&buf) + ruleBuf.WriteString(leftRuleStart) + ruleBuf.WriteByte('\n') + for sc.Scan() { + line := sc.Text() + prefix := leftRuleLine + if line == "" { + // Don't print the space after the line if there would be nothing + // after it anyway. + prefix = strings.TrimSpace(prefix) + } + ruleBuf.WriteString(prefix) + ruleBuf.WriteString(line) + ruleBuf.WriteByte('\n') + } + ruleBuf.WriteString(leftRuleEnd) + + return ruleBuf.String(), nil +} + +func (render *HumanRender) SourceSnippets(diag *diagnostic.Diagnostic) (string, error) { + if diag.Range == nil || diag.Snippet == nil { + // This should generally not happen, as long as sources are always + // loaded through the main loader. We may load things in other + // ways in weird cases, so we'll tolerate it at the expense of + // a not-so-helpful error message. + return fmt.Sprintf(" on %s line %d:\n (source code not available)\n", diag.Range.Filename, diag.Range.Start.Line), nil + } + + var ( + buf = new(bytes.Buffer) + snippet = diag.Snippet + code = snippet.Code + ) + + var contextStr string + if snippet.Context != "" { + contextStr = fmt.Sprintf(", in %s", snippet.Context) + } + if _, err := fmt.Fprintf(buf, " on %s line %d%s:\n", diag.Range.Filename, diag.Range.Start.Line, contextStr); err != nil { + return "", errors.WithStackTrace(err) + } + + // Split the snippet and render the highlighted section with underlines + start := snippet.HighlightStartOffset + end := snippet.HighlightEndOffset + + // Only buggy diagnostics can have an end range before the start, but + // we need to ensure we don't crash here if that happens. + if end < start { + end = start + 1 + if end > len(code) { + end = len(code) + } + } + + // If either start or end is out of range for the code buffer then + // we'll cap them at the bounds just to avoid a panic, although + // this would happen only if there's a bug in the code generating + // the snippet objects. + if start < 0 { + start = 0 + } else if start > len(code) { + start = len(code) + } + if end < 0 { + end = 0 + } else if end > len(code) { + end = len(code) + } + + before, highlight, after := code[0:start], code[start:end], code[end:] + code = fmt.Sprintf(render.colorize.Color("%s[underline][white]%s[reset]%s"), before, highlight, after) + + // Split the snippet into lines and render one at a time + lines := strings.Split(code, "\n") + for i, line := range lines { + if _, err := fmt.Fprintf( + buf, "%4d: %s\n", + snippet.StartLine+i, + line, + ); err != nil { + return "", errors.WithStackTrace(err) + } + } + + if len(snippet.Values) > 0 || (snippet.FunctionCall != nil && snippet.FunctionCall.Signature != nil) { + // The diagnostic may also have information about the dynamic + // values of relevant variables at the point of evaluation. + // This is particularly useful for expressions that get evaluated + // multiple times with different values, such as blocks using + // "count" and "for_each", or within "for" expressions. + values := make([]diagnostic.ExpressionValue, len(snippet.Values)) + copy(values, snippet.Values) + sort.Slice(values, func(i, j int) bool { + return values[i].Traversal < values[j].Traversal + }) + + fmt.Fprint(buf, render.colorize.Color(" [dark_gray]├────────────────[reset]\n")) + if callInfo := snippet.FunctionCall; callInfo != nil && callInfo.Signature != nil { + + if _, err := fmt.Fprintf(buf, render.colorize.Color(" [dark_gray]│[reset] while calling [bold]%s[reset]("), callInfo.CalledAs); err != nil { + return "", errors.WithStackTrace(err) + } + for i, param := range callInfo.Signature.Params { + if i > 0 { + buf.WriteString(", ") + } + buf.WriteString(param.Name) + } + if param := callInfo.Signature.VariadicParam; param != nil { + if len(callInfo.Signature.Params) > 0 { + buf.WriteString(", ") + } + buf.WriteString(param.Name) + buf.WriteString("...") + } + buf.WriteString(")\n") + } + for _, value := range values { + if _, err := fmt.Fprintf(buf, render.colorize.Color(" [dark_gray]│[reset] [bold]%s[reset] %s\n"), value.Traversal, value.Statement); err != nil { + return "", errors.WithStackTrace(err) + } + } + } + buf.WriteByte('\n') + + return buf.String(), nil +} diff --git a/internal/view/json_render.go b/internal/view/json_render.go new file mode 100644 index 000000000..4302546c9 --- /dev/null +++ b/internal/view/json_render.go @@ -0,0 +1,36 @@ +package view + +import ( + "encoding/json" + + "github.com/gruntwork-io/go-commons/errors" + "github.com/gruntwork-io/terragrunt/internal/view/diagnostic" +) + +type JSONRender struct{} + +func NewJSONRender() Render { + return &JSONRender{} +} + +func (render *JSONRender) Diagnostics(diags diagnostic.Diagnostics) (string, error) { + return render.toJSON(diags) +} + +func (render *JSONRender) InvalidConfigPath(filenames []string) (string, error) { + return render.toJSON(filenames) +} + +func (render *JSONRender) toJSON(val any) (string, error) { + jsonBytes, err := json.Marshal(val) + if err != nil { + return "", errors.WithStackTrace(err) + } + + if len(jsonBytes) == 0 { + return "", nil + } + + jsonBytes = append(jsonBytes, '\n') + return string(jsonBytes), nil +} diff --git a/internal/view/writer.go b/internal/view/writer.go new file mode 100644 index 000000000..8e714ea7d --- /dev/null +++ b/internal/view/writer.go @@ -0,0 +1,64 @@ +package view + +import ( + "fmt" + "io" + + "github.com/gruntwork-io/go-commons/errors" + "github.com/gruntwork-io/terragrunt/internal/view/diagnostic" + "github.com/gruntwork-io/terragrunt/util" +) + +type Render interface { + // Diagnostics renders early diagnostics, resulting from argument parsing. + Diagnostics(diags diagnostic.Diagnostics) (string, error) + + // InvalidConfigPath renders paths to configurations that contain errors. + InvalidConfigPath(filenames []string) (string, error) +} + +// Writer is the base layer for command views, encapsulating a set of I/O streams, a colorize implementation, and implementing a human friendly view for diagnostics. +type Writer struct { + io.Writer + render Render +} + +func NewWriter(writer io.Writer, render Render) *Writer { + return &Writer{ + Writer: writer, + render: render, + } +} + +func (writer *Writer) Diagnostics(diags diagnostic.Diagnostics) error { + output, err := writer.render.Diagnostics(diags) + if err != nil { + return err + } + + return writer.output(output) +} + +func (writer *Writer) InvalidConfigPath(diags diagnostic.Diagnostics) error { + var filenames []string + + for _, diag := range diags { + if diag.Range != nil && diag.Range.Filename != "" && !util.ListContainsElement(filenames, diag.Range.Filename) { + filenames = append(filenames, diag.Range.Filename) + } + } + + output, err := writer.render.InvalidConfigPath(filenames) + if err != nil { + return err + } + + return writer.output(output) +} + +func (writer *Writer) output(output string) error { + if _, err := fmt.Fprint(writer, output); err != nil { + return errors.WithStackTrace(err) + } + return nil +} diff --git a/options/options.go b/options/options.go index b5d4ed57a..6c4f15f1c 100644 --- a/options/options.go +++ b/options/options.go @@ -306,6 +306,9 @@ type TerragruntOptions struct { // The command and arguments that can be used to fetch authentication configurations. // Terragrunt invokes this command before running tofu/terraform operations for each working directory. AuthProviderCmd string + + // Allows to skip the output of all dependencies. Intended for use with `hclvalidate` command. + SkipOutput bool } // TerragruntOptionsFunc is a functional option type used to pass options in certain integration tests @@ -553,6 +556,7 @@ func (opts *TerragruntOptions) Clone(terragruntConfigPath string) *TerragruntOpt OutputFolder: opts.OutputFolder, JsonOutputFolder: opts.JsonOutputFolder, AuthProviderCmd: opts.AuthProviderCmd, + SkipOutput: opts.SkipOutput, } } diff --git a/test/fixture-hclvalidate/first/b/terragrunt.hcl b/test/fixture-hclvalidate/first/b/terragrunt.hcl new file mode 100644 index 000000000..328970f64 --- /dev/null +++ b/test/fixture-hclvalidate/first/b/terragrunt.hcl @@ -0,0 +1,3 @@ +dependency "a" { + config_path = "${path_relative_from_include()}/${path_relative_to_include()}/../a" +} diff --git a/test/fixture-hclvalidate/second/a/main.tf b/test/fixture-hclvalidate/second/a/main.tf new file mode 100644 index 000000000..ea1582209 --- /dev/null +++ b/test/fixture-hclvalidate/second/a/main.tf @@ -0,0 +1,8 @@ +variable "a" { + type = string + default = "a" +} + +output "a" { + value = var.a +} diff --git a/test/fixture-hclvalidate/second/a/terragrunt.hcl b/test/fixture-hclvalidate/second/a/terragrunt.hcl new file mode 100644 index 000000000..bd4dfb158 --- /dev/null +++ b/test/fixture-hclvalidate/second/a/terragrunt.hcl @@ -0,0 +1,3 @@ +locals { + t = +} diff --git a/test/fixture-hclvalidate/second/c/main.tf b/test/fixture-hclvalidate/second/c/main.tf new file mode 100644 index 000000000..0aa6da576 --- /dev/null +++ b/test/fixture-hclvalidate/second/c/main.tf @@ -0,0 +1,8 @@ +variable "c" { + type = string + default = "c" +} + +output "c" { + value = var.c +} diff --git a/test/fixture-hclvalidate/second/c/terragrunt.hcl b/test/fixture-hclvalidate/second/c/terragrunt.hcl new file mode 100644 index 000000000..e1a9d5ede --- /dev/null +++ b/test/fixture-hclvalidate/second/c/terragrunt.hcl @@ -0,0 +1,13 @@ +include "b" { + path = "../../first/b/terragrunt.hcl" +} + +inputs = { + c = dependency.a.outputs.z +} + +locals { + vvv = dependency.a.outputs.z + + ddd = dependency.d +} diff --git a/test/fixture-hclvalidate/second/d/main.tf b/test/fixture-hclvalidate/second/d/main.tf new file mode 100644 index 000000000..95ee55772 --- /dev/null +++ b/test/fixture-hclvalidate/second/d/main.tf @@ -0,0 +1,8 @@ +variabl "d" { + type = string + default = "d" +} + +output "d" { + value = var.d +} diff --git a/test/fixture-hclvalidate/second/d/terragrunt.hcl b/test/fixture-hclvalidate/second/d/terragrunt.hcl new file mode 100644 index 000000000..0b7b85d02 --- /dev/null +++ b/test/fixture-hclvalidate/second/d/terragrunt.hcl @@ -0,0 +1,11 @@ +dependency "c" { + config_path = "../c" + + mock_outputs = { + c = "mocked-c" + } +} + +inputs = { + d = dependency.c.outputs.c +} diff --git a/test/fixutre-excludes-file/b/terragrunt_rendered.json b/test/fixutre-excludes-file/b/terragrunt_rendered.json deleted file mode 100644 index 3c0ad8e4b..000000000 --- a/test/fixutre-excludes-file/b/terragrunt_rendered.json +++ /dev/null @@ -1 +0,0 @@ -{"dependencies":null,"download_dir":"","generate":{},"iam_assume_role_duration":null,"iam_assume_role_session_name":"","iam_role":"","iam_web_identity_token":"","inputs":null,"locals":null,"retry_max_attempts":null,"retry_sleep_interval_sec":null,"retryable_errors":null,"skip":false,"terraform_binary":"","terraform_version_constraint":"","terragrunt_version_constraint":""} \ No newline at end of file diff --git a/test/fixutre-excludes-file/d/terragrunt_rendered.json b/test/fixutre-excludes-file/d/terragrunt_rendered.json deleted file mode 100644 index 3c0ad8e4b..000000000 --- a/test/fixutre-excludes-file/d/terragrunt_rendered.json +++ /dev/null @@ -1 +0,0 @@ -{"dependencies":null,"download_dir":"","generate":{},"iam_assume_role_duration":null,"iam_assume_role_session_name":"","iam_role":"","iam_web_identity_token":"","inputs":null,"locals":null,"retry_max_attempts":null,"retry_sleep_interval_sec":null,"retryable_errors":null,"skip":false,"terraform_binary":"","terraform_version_constraint":"","terragrunt_version_constraint":""} \ No newline at end of file diff --git a/test/integration_catalog_test.go b/test/integration_catalog_test.go index e452ce80b..ac17dad90 100644 --- a/test/integration_catalog_test.go +++ b/test/integration_catalog_test.go @@ -117,7 +117,7 @@ func readConfig(t *testing.T, opts *options.TerragruntOptions) *config.Terragrun opts, err := options.NewTerragruntOptionsForTest(filepath.Join(opts.WorkingDir, "terragrunt.hcl")) assert.NoError(t, err) - cfg, err := config.ReadTerragruntConfig(opts) + cfg, err := config.ReadTerragruntConfig(context.Background(), opts, config.DefaultParserOptions(opts)) assert.NoError(t, err) return cfg diff --git a/test/integration_serial_test.go b/test/integration_serial_test.go index 95320fe2a..f7d3c6edf 100644 --- a/test/integration_serial_test.go +++ b/test/integration_serial_test.go @@ -29,8 +29,6 @@ import ( "github.com/gruntwork-io/terragrunt/util" ) -// @SONAR_STOP@ - // NOTE: We don't run these tests in parallel because it modifies the environment variable, so it can affect other tests func TestTerragruntProviderCacheWithFilesystemMirror(t *testing.T) { @@ -870,5 +868,3 @@ func TestReadTerragruntAuthProviderCmdRemoteState(t *testing.T) { runTerragrunt(t, fmt.Sprintf("terragrunt plan --terragrunt-non-interactive --terragrunt-working-dir %s --terragrunt-auth-provider-cmd %s", rootPath, mockAuthCmd)) } - -// @SONAR_START@ diff --git a/test/integration_test.go b/test/integration_test.go index 5a842ec81..4ecf7d38b 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -48,19 +48,19 @@ import ( "github.com/gruntwork-io/terragrunt/codegen" "github.com/gruntwork-io/terragrunt/config" terragruntDynamoDb "github.com/gruntwork-io/terragrunt/dynamodb" + "github.com/gruntwork-io/terragrunt/internal/view/diagnostic" "github.com/gruntwork-io/terragrunt/options" "github.com/gruntwork-io/terragrunt/remote" "github.com/gruntwork-io/terragrunt/shell" "github.com/gruntwork-io/terragrunt/util" ) -// @SONAR_STOP@ - // hard-code this to match the test fixture for now const ( TERRAFORM_REMOTE_STATE_S3_REGION = "us-west-2" TERRAFORM_REMOTE_STATE_GCP_REGION = "eu" TEST_FIXTURE_PATH = "fixture/" + TEST_FIXTURE_HCLVALIDATE = "fixture-hclvalidate" TEST_FIXTURE_EXCLUDES_FILE = "fixutre-excludes-file" TEST_FIXTURE_INIT_ONCE = "fixture-init-once" TEST_FIXTURE_PROVIDER_CACHE_MULTIPLE_PLATFORMS = "fixture-provider-cache/multiple-platforms" @@ -251,6 +251,117 @@ func TestTerragruntExcludesFile(t *testing.T) { } } +func TestHclvalidateDiagnostic(t *testing.T) { + t.Parallel() + + cleanupTerraformFolder(t, TEST_FIXTURE_HCLVALIDATE) + tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_HCLVALIDATE) + rootPath := util.JoinPath(tmpEnvPath, TEST_FIXTURE_HCLVALIDATE) + + expectedDiags := diagnostic.Diagnostics{ + &diagnostic.Diagnostic{ + Severity: diagnostic.DiagnosticSeverity(hcl.DiagError), + Summary: "Invalid expression", + Detail: "Expected the start of an expression, but found an invalid expression token.", + Range: &diagnostic.Range{ + Filename: filepath.Join(rootPath, "second/a/terragrunt.hcl"), + Start: diagnostic.Pos{Line: 2, Column: 6, Byte: 14}, + End: diagnostic.Pos{Line: 3, Column: 1, Byte: 15}, + }, + Snippet: &diagnostic.Snippet{ + Context: "locals", + Code: " t =\n}", + StartLine: 2, + HighlightStartOffset: 5, + HighlightEndOffset: 6, + }, + }, + &diagnostic.Diagnostic{ + Severity: diagnostic.DiagnosticSeverity(hcl.DiagError), + Summary: "Can't evaluate expression", + Detail: "You can only reference to other local variables here, but it looks like you're referencing something else (\"dependency\" is not defined)", + Range: &diagnostic.Range{ + Filename: filepath.Join(rootPath, "second/c/terragrunt.hcl"), + Start: diagnostic.Pos{Line: 10, Column: 9, Byte: 117}, + End: diagnostic.Pos{Line: 10, Column: 31, Byte: 139}, + }, + Snippet: &diagnostic.Snippet{ + Context: "locals", + Code: " vvv = dependency.a.outputs.z", + StartLine: 10, + HighlightStartOffset: 8, + HighlightEndOffset: 30, + }, + }, + &diagnostic.Diagnostic{ + Severity: diagnostic.DiagnosticSeverity(hcl.DiagError), + Summary: "Can't evaluate expression", + Detail: "You can only reference to other local variables here, but it looks like you're referencing something else (\"dependency\" is not defined)", + Range: &diagnostic.Range{ + Filename: filepath.Join(rootPath, "second/c/terragrunt.hcl"), + Start: diagnostic.Pos{Line: 12, Column: 9, Byte: 149}, + End: diagnostic.Pos{Line: 12, Column: 21, Byte: 161}, + }, + Snippet: &diagnostic.Snippet{ + Context: "locals", + Code: " ddd = dependency.d", + StartLine: 12, + HighlightStartOffset: 8, + HighlightEndOffset: 20, + }, + }, + &diagnostic.Diagnostic{ + Severity: diagnostic.DiagnosticSeverity(hcl.DiagError), + Summary: "Unsupported attribute", + Detail: "This object does not have an attribute named \"outputs\".", + Range: &diagnostic.Range{ + Filename: filepath.Join(rootPath, "second/c/terragrunt.hcl"), + Start: diagnostic.Pos{Line: 6, Column: 19, Byte: 86}, + End: diagnostic.Pos{Line: 6, Column: 27, Byte: 94}, + }, + Snippet: &diagnostic.Snippet{ + Context: "", + Code: " c = dependency.a.outputs.z", + StartLine: 6, + HighlightStartOffset: 18, + HighlightEndOffset: 26, + Values: []diagnostic.ExpressionValue{diagnostic.ExpressionValue{Traversal: "dependency.a", Statement: "is object with no attributes"}}, + }, + }, + } + + stdout, _, err := runTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt hclvalidate --terragrunt-working-dir %s --terragrunt-hclvalidate-json", rootPath)) + require.NoError(t, err) + + var actualDiags diagnostic.Diagnostics + + err = json.Unmarshal([]byte(strings.TrimSpace(stdout)), &actualDiags) + require.NoError(t, err) + + assert.ElementsMatch(t, expectedDiags, actualDiags) +} + +func TestHclvalidateInvalidConfigPath(t *testing.T) { + cleanupTerraformFolder(t, TEST_FIXTURE_HCLVALIDATE) + tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_HCLVALIDATE) + rootPath := util.JoinPath(tmpEnvPath, TEST_FIXTURE_HCLVALIDATE) + + expectedPaths := []string{ + filepath.Join(rootPath, "second/a/terragrunt.hcl"), + filepath.Join(rootPath, "second/c/terragrunt.hcl"), + } + + stdout, _, err := runTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt hclvalidate --terragrunt-working-dir %s --terragrunt-hclvalidate-json --terragrunt-hclvalidate-invalid", rootPath)) + require.NoError(t, err) + + var actualPaths []string + + err = json.Unmarshal([]byte(strings.TrimSpace(stdout)), &actualPaths) + require.NoError(t, err) + + assert.ElementsMatch(t, expectedPaths, actualPaths) +} + func TestTerragruntProviderCacheMultiplePlatforms(t *testing.T) { t.Parallel() @@ -7266,5 +7377,3 @@ func findFilesWithExtension(dir string, ext string) ([]string, error) { return files, err } - -// @SONAR_START@ diff --git a/util/logger.go b/util/logger.go index 471744782..2a855e235 100644 --- a/util/logger.go +++ b/util/logger.go @@ -96,8 +96,8 @@ func CreateLogEntryWithWriter(writer io.Writer, prefix string, level logrus.Leve } // GetDiagnosticsWriter returns a hcl2 parsing diagnostics emitter for the current terminal. -func GetDiagnosticsWriter(logger *logrus.Entry, parser *hclparse.Parser) hcl.DiagnosticWriter { - termColor := term.IsTerminal(int(os.Stderr.Fd())) +func GetDiagnosticsWriter(logger *logrus.Entry, parser *hclparse.Parser, disableColor bool) hcl.DiagnosticWriter { + termColor := !disableColor && term.IsTerminal(int(os.Stderr.Fd())) termWidth, _, err := term.GetSize(int(os.Stdout.Fd())) if err != nil { termWidth = 80 @@ -113,7 +113,6 @@ func GetDefaultLogLevel() logrus.Level { if defaultLogLevelStr == "" { return defaultLogLevel } - return ParseLogLevel(defaultLogLevelStr) }