From 6f5e4488107f306083322765bdb74dd23631ed39 Mon Sep 17 00:00:00 2001 From: Rodrigo Fior Kuntzer Date: Tue, 24 Sep 2024 16:07:16 +0200 Subject: [PATCH] feat: introducing "copy_terraform_lock_file" to fine tune Lock File Handling (#2889) * feat: introducing "copy_terraform_lock_file" to fine tune Lock File Handling Signed-off-by: Rodrigo Fior Kuntzer * fix: renaming test name Signed-off-by: Rodrigo Fior Kuntzer * fix: linting doc files Signed-off-by: Rodrigo Fior Kuntzer * docs: applying suggestions Signed-off-by: Rodrigo Fior Kuntzer * fix: adding comment about early return Signed-off-by: Rodrigo Fior Kuntzer * fix: aligning code with the latest changes from main Signed-off-by: Rodrigo Fior Kuntzer * test: fixing the TestRenderJsonMetadataTerraform test Signed-off-by: Rodrigo Fior Kuntzer * docs: applying suggestions Signed-off-by: Rodrigo Fior Kuntzer --------- Signed-off-by: Rodrigo Fior Kuntzer --- cli/commands/terraform/action.go | 13 +++- cli/commands/terraform/action_test.go | 77 +++++++++++++++++++ config/config.go | 2 + config/config_as_cty.go | 26 ++++--- config/include.go | 8 ++ config/include_test.go | 11 +++ docs/_docs/02_features/lock-file-handling.md | 13 ++++ .../config-blocks-and-attributes.md | 5 ++ .../terragrunt.hcl | 8 ++ .../module-a/terragrunt.hcl | 7 ++ .../module-b/terragrunt.hcl | 14 ++++ .../terragrunt.hcl | 7 ++ test/fixtures/read-config/full/source.hcl | 5 +- test/integration_download_test.go | 30 ++++++++ test/integration_json_test.go | 13 ++-- test/integration_test.go | 5 +- 16 files changed, 220 insertions(+), 24 deletions(-) create mode 100644 test/fixtures/download/local-disable-copy-terraform-lock-file/terragrunt.hcl create mode 100644 test/fixtures/download/local-include-disable-copy-lock-file/module-a/terragrunt.hcl create mode 100644 test/fixtures/download/local-include-disable-copy-lock-file/module-b/terragrunt.hcl create mode 100644 test/fixtures/download/local-include-disable-copy-lock-file/terragrunt.hcl diff --git a/cli/commands/terraform/action.go b/cli/commands/terraform/action.go index 31e27f236..cc645fde7 100644 --- a/cli/commands/terraform/action.go +++ b/cli/commands/terraform/action.go @@ -327,7 +327,7 @@ func runTerragruntWithConfig(ctx context.Context, originalTerragruntOptions *opt runTerraformError := RunTerraformWithRetry(ctx, terragruntOptions) var lockFileError error - if shouldCopyLockFile(terragruntOptions.TerraformCliArgs) { + if ShouldCopyLockFile(terragruntOptions.TerraformCliArgs, terragruntConfig.Terraform) { // Copy the lock file from the Terragrunt working dir (e.g., .terragrunt-cache/xxx/) to the // user's working dir (e.g., /live/stage/vpc). That way, the lock file will end up right next to the user's // terragrunt.hcl and can be checked into version control. Note that in the past, Terragrunt allowed the @@ -372,6 +372,7 @@ func confirmActionWithDependentModules(ctx context.Context, terragruntOptions *o return true } +// ShouldCopyLockFile verifies if the lock file should be copied to the user's working directory // Terraform 0.14 now manages a lock file for providers. This can be updated // in three ways: // * `terraform init` in a module where no `.terraform.lock.hcl` exists @@ -386,7 +387,15 @@ func confirmActionWithDependentModules(ctx context.Context, terragruntOptions *o // There are lots of details at [hashicorp/terraform#27264](https://github.com/hashicorp/terraform/issues/27264#issuecomment-743389837) // The `providers lock` sub command enables you to ensure that the lock file is // fully populated. -func shouldCopyLockFile(args []string) bool { +func ShouldCopyLockFile(args []string, terraformConfig *config.TerraformConfig) bool { + // If the user has explicitly set CopyTerraformLockFile to false, then we should not copy the lock file on any command + // This is useful for users who want to manage the lock file themselves outside the working directory + // if the user has not set CopyTerraformLockFile or if they have explicitly defined it to true, + // then we should copy the lock file on init and providers lock as defined above and not do and early return here + if terraformConfig != nil && terraformConfig.CopyTerraformLockFile != nil && !*terraformConfig.CopyTerraformLockFile { + return false + } + if util.FirstArg(args) == terraform.CommandNameInit { return true } diff --git a/cli/commands/terraform/action_test.go b/cli/commands/terraform/action_test.go index ccc18acbb..dec4847f8 100644 --- a/cli/commands/terraform/action_test.go +++ b/cli/commands/terraform/action_test.go @@ -498,3 +498,80 @@ func createTempFile(t *testing.T) string { return filepath.ToSlash(tmpFile.Name()) } + +func TestShouldCopyLockFile(t *testing.T) { + t.Parallel() + + type args struct { + args []string + terraformConfig *config.TerraformConfig + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "init without terraform config", + args: args{ + args: []string{"init"}, + }, + want: true, + }, + { + name: "providers lock without terraform config", + args: args{ + args: []string{"providers", "lock"}, + }, + want: true, + }, + { + name: "providers schema without terraform config", + args: args{ + args: []string{"providers", "schema"}, + }, + want: false, + }, + { + name: "plan without terraform config", + args: args{ + args: []string{"plan"}, + }, + want: false, + }, + { + name: "init with empty terraform config", + args: args{ + args: []string{"init"}, + terraformConfig: &config.TerraformConfig{}, + }, + want: true, + }, + { + name: "init with CopyTerraformLockFile enabled", + args: args{ + args: []string{"init"}, + terraformConfig: &config.TerraformConfig{ + CopyTerraformLockFile: &[]bool{true}[0], + }, + }, + want: true, + }, + { + name: "init with CopyTerraformLockFile disabled", + args: args{ + args: []string{"init"}, + terraformConfig: &config.TerraformConfig{ + CopyTerraformLockFile: &[]bool{false}[0], + }, + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + assert.Equalf(t, tt.want, terraform.ShouldCopyLockFile(tt.args.args, tt.args.terraformConfig), "shouldCopyLockFile(%v, %v)", tt.args.args, tt.args.terraformConfig) + }) + } +} diff --git a/config/config.go b/config/config.go index 783495d77..fb8923fcc 100644 --- a/config/config.go +++ b/config/config.go @@ -438,6 +438,8 @@ type TerraformConfig struct { // Ideally we can avoid the pointer to list slice, but if it is not a pointer, Terraform requires the attribute to // be defined and we want to make this optional. IncludeInCopy *[]string `hcl:"include_in_copy,attr"` + + CopyTerraformLockFile *bool `hcl:"copy_terraform_lock_file,attr"` } func (cfg *TerraformConfig) String() string { diff --git a/config/config_as_cty.go b/config/config_as_cty.go index 2bca488cf..d926265d9 100644 --- a/config/config_as_cty.go +++ b/config/config_as_cty.go @@ -495,12 +495,13 @@ func engineConfigAsCty(config *EngineConfig) (cty.Value, error) { // CtyTerraformConfig is an alternate representation of TerraformConfig that converts internal blocks into a map that // maps the name to the underlying struct, as opposed to a list representation. type CtyTerraformConfig struct { - ExtraArgs map[string]TerraformExtraArguments `cty:"extra_arguments"` - Source *string `cty:"source"` - IncludeInCopy *[]string `cty:"include_in_copy"` - BeforeHooks map[string]Hook `cty:"before_hook"` - AfterHooks map[string]Hook `cty:"after_hook"` - ErrorHooks map[string]ErrorHook `cty:"error_hook"` + ExtraArgs map[string]TerraformExtraArguments `cty:"extra_arguments"` + Source *string `cty:"source"` + IncludeInCopy *[]string `cty:"include_in_copy"` + CopyTerraformLockFile *bool `cty:"copy_terraform_lock_file"` + BeforeHooks map[string]Hook `cty:"before_hook"` + AfterHooks map[string]Hook `cty:"after_hook"` + ErrorHooks map[string]ErrorHook `cty:"error_hook"` } // Serialize TerraformConfig to a cty Value, but with maps instead of lists for the blocks. @@ -510,12 +511,13 @@ func terraformConfigAsCty(config *TerraformConfig) (cty.Value, error) { } configCty := CtyTerraformConfig{ - Source: config.Source, - IncludeInCopy: config.IncludeInCopy, - ExtraArgs: map[string]TerraformExtraArguments{}, - BeforeHooks: map[string]Hook{}, - AfterHooks: map[string]Hook{}, - ErrorHooks: map[string]ErrorHook{}, + Source: config.Source, + IncludeInCopy: config.IncludeInCopy, + CopyTerraformLockFile: config.CopyTerraformLockFile, + ExtraArgs: map[string]TerraformExtraArguments{}, + BeforeHooks: map[string]Hook{}, + AfterHooks: map[string]Hook{}, + ErrorHooks: map[string]ErrorHook{}, } for _, arg := range config.ExtraArgs { diff --git a/config/include.go b/config/include.go index 94a6203e1..84cdd1770 100644 --- a/config/include.go +++ b/config/include.go @@ -278,6 +278,10 @@ func (cfg *TerragruntConfig) Merge(sourceConfig *TerragruntConfig, terragruntOpt cfg.Terraform.Source = sourceConfig.Terraform.Source } + if sourceConfig.Terraform.CopyTerraformLockFile != nil { + cfg.Terraform.CopyTerraformLockFile = sourceConfig.Terraform.CopyTerraformLockFile + } + mergeExtraArgs(terragruntOptions, sourceConfig.Terraform.ExtraArgs, &cfg.Terraform.ExtraArgs) mergeHooks(terragruntOptions, sourceConfig.Terraform.BeforeHooks, &cfg.Terraform.BeforeHooks) @@ -446,6 +450,10 @@ func (cfg *TerragruntConfig) DeepMerge(sourceConfig *TerragruntConfig, terragrun cfg.Terraform.Source = sourceConfig.Terraform.Source } + if sourceConfig.Terraform.CopyTerraformLockFile != nil { + cfg.Terraform.CopyTerraformLockFile = sourceConfig.Terraform.CopyTerraformLockFile + } + if sourceConfig.Terraform.IncludeInCopy != nil { srcList := *sourceConfig.Terraform.IncludeInCopy diff --git a/config/include_test.go b/config/include_test.go index 6f9c9e0c9..2ee641355 100644 --- a/config/include_test.go +++ b/config/include_test.go @@ -144,6 +144,11 @@ func TestMergeConfigIntoIncludedConfig(t *testing.T) { &config.TerragruntConfig{IamWebIdentityToken: "token"}, &config.TerragruntConfig{IamWebIdentityToken: "token"}, }, + { + &config.TerragruntConfig{Terraform: &config.TerraformConfig{CopyTerraformLockFile: &[]bool{false}[0]}}, + &config.TerragruntConfig{Terraform: &config.TerraformConfig{IncludeInCopy: &[]string{"abc"}}}, + &config.TerragruntConfig{Terraform: &config.TerraformConfig{CopyTerraformLockFile: &[]bool{false}[0], IncludeInCopy: &[]string{"abc"}}}, + }, } for _, testCase := range testCases { @@ -288,6 +293,12 @@ func TestDeepMergeConfigIntoIncludedConfig(t *testing.T) { &config.TerragruntConfig{Inputs: originalMap}, &config.TerragruntConfig{Inputs: mergedMap}, }, + { + "terraform copy_terraform_lock_file", + &config.TerragruntConfig{Terraform: &config.TerraformConfig{CopyTerraformLockFile: &[]bool{false}[0]}}, + &config.TerragruntConfig{Terraform: &config.TerraformConfig{IncludeInCopy: &[]string{"abc"}}}, + &config.TerragruntConfig{Terraform: &config.TerraformConfig{CopyTerraformLockFile: &[]bool{false}[0], IncludeInCopy: &[]string{"abc"}}}, + }, } for _, tt := range tc { diff --git a/docs/_docs/02_features/lock-file-handling.md b/docs/_docs/02_features/lock-file-handling.md index 0dbbdda57..ba886f2d9 100644 --- a/docs/_docs/02_features/lock-file-handling.md +++ b/docs/_docs/02_features/lock-file-handling.md @@ -96,3 +96,16 @@ should end up looking something like this: Also, any time you change the providers you're using, and re-run `init`, the lock file will be updated, so make sure to check the updates into version control too. + +### Disabling the copy of the generated lock file + +In certain use cases, like when using a remote module containing a lock file within it, you probably +don't want Terragrunt to also copy the lock file into your working directory. In these scenarios you, can opt-out of copying +the `.terraform.lock.hcl` file by using `copy_terraform_lock_file = false` in the `terraform` configuration block as follows: + +```hcl +terraform { + ... + copy_terraform_lock_file = false +} +``` diff --git a/docs/_docs/04_reference/config-blocks-and-attributes.md b/docs/_docs/04_reference/config-blocks-and-attributes.md index bcc5de373..533a70241 100644 --- a/docs/_docs/04_reference/config-blocks-and-attributes.md +++ b/docs/_docs/04_reference/config-blocks-and-attributes.md @@ -105,6 +105,11 @@ The `terraform` block supports the following arguments: can specify that in this list to ensure it gets copied over to the scratch copy (e.g., `include_in_copy = [".python-version"]`). +- `copy_terraform_lock_file` (attribute): In certain use cases, you don't want to check the terraform provider lock + file into your source repository from your working directory as described in + [Lock File Handling]({{site.baseurl}}/docs/features/lock-file-handling/). This attribute allows you to disable the copy + of the generated or existing `.terraform.lock.hcl` from the temp folder into the working directory. Default is `true`. + - `extra_arguments` (block): Nested blocks used to specify extra CLI arguments to pass to the `tofu`/`terraform` binary. Learn more about its usage in the [Keep your CLI flags DRY]({{site.baseurl}}/docs/features/keep-your-cli-flags-dry/) use case overview. Supports the following arguments: diff --git a/test/fixtures/download/local-disable-copy-terraform-lock-file/terragrunt.hcl b/test/fixtures/download/local-disable-copy-terraform-lock-file/terragrunt.hcl new file mode 100644 index 000000000..dd910dc33 --- /dev/null +++ b/test/fixtures/download/local-disable-copy-terraform-lock-file/terragrunt.hcl @@ -0,0 +1,8 @@ +inputs = { + name = "World" +} + +terraform { + source = "../hello-world" + copy_terraform_lock_file = false +} diff --git a/test/fixtures/download/local-include-disable-copy-lock-file/module-a/terragrunt.hcl b/test/fixtures/download/local-include-disable-copy-lock-file/module-a/terragrunt.hcl new file mode 100644 index 000000000..077fe2a7c --- /dev/null +++ b/test/fixtures/download/local-include-disable-copy-lock-file/module-a/terragrunt.hcl @@ -0,0 +1,7 @@ +inputs = { + name = "Module A" +} + +terraform { + source = "../../hello-world" +} diff --git a/test/fixtures/download/local-include-disable-copy-lock-file/module-b/terragrunt.hcl b/test/fixtures/download/local-include-disable-copy-lock-file/module-b/terragrunt.hcl new file mode 100644 index 000000000..e2376d830 --- /dev/null +++ b/test/fixtures/download/local-include-disable-copy-lock-file/module-b/terragrunt.hcl @@ -0,0 +1,14 @@ +inputs = { + name = "Module B" +} + +terraform { + source = "../../hello-world" + copy_terraform_lock_file = false +} + +prevent_destroy = true + +include { + path = find_in_parent_folders("terragrunt.hcl") +} \ No newline at end of file diff --git a/test/fixtures/download/local-include-disable-copy-lock-file/terragrunt.hcl b/test/fixtures/download/local-include-disable-copy-lock-file/terragrunt.hcl new file mode 100644 index 000000000..3c8af1a10 --- /dev/null +++ b/test/fixtures/download/local-include-disable-copy-lock-file/terragrunt.hcl @@ -0,0 +1,7 @@ +terraform { + include_in_copy = ["**/.terraform-version"] +} + +dependencies { + paths = ["../module-a"] +} \ No newline at end of file diff --git a/test/fixtures/read-config/full/source.hcl b/test/fixtures/read-config/full/source.hcl index d927f3da6..0835d4469 100644 --- a/test/fixtures/read-config/full/source.hcl +++ b/test/fixtures/read-config/full/source.hcl @@ -24,8 +24,9 @@ remote_state { } terraform { - source = "./delorean" - include_in_copy = ["time_machine.*"] + source = "./delorean" + include_in_copy = ["time_machine.*"] + copy_terraform_lock_file = true extra_arguments "var-files" { commands = ["apply", "plan"] diff --git a/test/integration_download_test.go b/test/integration_download_test.go index 4aa99beaa..64f90395e 100644 --- a/test/integration_download_test.go +++ b/test/integration_download_test.go @@ -39,6 +39,8 @@ const ( testFixtureLocalPreventDestroyDependencies = "fixtures/download/local-with-prevent-destroy-dependencies" testFixtureLocalIncludePreventDestroyDependencies = "fixtures/download/local-include-with-prevent-destroy-dependencies" testFixtureNotExistingSource = "fixtures/download/invalid-path" + testFixtureDisableCopyLockFilePath = "fixtures/download/local-disable-copy-terraform-lock-file" + testFixtureIncludeDisableCopyLockFilePath = "fixtures/download/local-include-disable-copy-lock-file/module-b" ) func TestLocalDownload(t *testing.T) { @@ -55,6 +57,34 @@ func TestLocalDownload(t *testing.T) { runTerragrunt(t, "terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir "+testFixtureLocalDownloadPath) } +func TestLocalDownloadDisableCopyTerraformLockFile(t *testing.T) { + t.Parallel() + + cleanupTerraformFolder(t, testFixtureDisableCopyLockFilePath) + + runTerragrunt(t, "terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir "+testFixtureDisableCopyLockFilePath) + + // The terraform lock file should not be copied if `copy_terraform_lock_file = false` + assert.NoFileExists(t, util.JoinPath(testFixtureDisableCopyLockFilePath, util.TerraformLockFile)) + + // Run a second time to make sure the temporary folder can be reused without errors + runTerragrunt(t, "terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir "+testFixtureDisableCopyLockFilePath) +} + +func TestLocalIncludeDisableCopyTerraformLockFile(t *testing.T) { + t.Parallel() + + cleanupTerraformFolder(t, testFixtureIncludeDisableCopyLockFilePath) + + runTerragrunt(t, "terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir "+testFixtureIncludeDisableCopyLockFilePath) + + // The terraform lock file should not be copied if `copy_terraform_lock_file = false` + assert.NoFileExists(t, util.JoinPath(testFixtureIncludeDisableCopyLockFilePath, util.TerraformLockFile)) + + // Run a second time to make sure the temporary folder can be reused without errors + runTerragrunt(t, "terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir "+testFixtureIncludeDisableCopyLockFilePath) +} + func TestLocalDownloadWithHiddenFolder(t *testing.T) { t.Parallel() diff --git a/test/integration_json_test.go b/test/integration_json_test.go index f5a7353cf..c33c9f53c 100644 --- a/test/integration_json_test.go +++ b/test/integration_json_test.go @@ -429,12 +429,13 @@ func TestRenderJsonMetadataTerraform(t *testing.T) { var expectedTerraform = map[string]interface{}{ "metadata": terragruntMetadata, "value": map[string]interface{}{ - "after_hook": map[string]interface{}{}, - "before_hook": map[string]interface{}{}, - "error_hook": map[string]interface{}{}, - "extra_arguments": map[string]interface{}{}, - "include_in_copy": nil, - "source": "../terraform", + "after_hook": map[string]interface{}{}, + "before_hook": map[string]interface{}{}, + "error_hook": map[string]interface{}{}, + "extra_arguments": map[string]interface{}{}, + "include_in_copy": nil, + "source": "../terraform", + "copy_terraform_lock_file": nil, }, } diff --git a/test/integration_test.go b/test/integration_test.go index e524e5088..7fe118f27 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -2043,8 +2043,9 @@ func TestReadTerragruntConfigFull(t *testing.T) { assert.Equal( t, map[string]interface{}{ - "source": "./delorean", - "include_in_copy": []interface{}{"time_machine.*"}, + "source": "./delorean", + "include_in_copy": []interface{}{"time_machine.*"}, + "copy_terraform_lock_file": true, "extra_arguments": map[string]interface{}{ "var-files": map[string]interface{}{ "name": "var-files",