From 4a6ce03c4eebc02ebd81641bc2158712b166a8db Mon Sep 17 00:00:00 2001 From: Sunny Date: Fri, 5 Jan 2024 16:48:29 +0000 Subject: [PATCH 1/2] Add testenv.Destroy for deleting resources testenv.Destroy() can be used to delete resources from a terraform configuration directory without going through the whole infrastructure set up steps. This is to be used in CI to always run at the end to make sure that everything has been destroyed after the test run. In ideal case, this won't be needed, as the environment constructor, testenv.New(), has signal handlers to gracefully delete the infrastructure. But if the whole test process gets terminated, a new process can just run testenv.Destroy() to perform the cleanup. Signed-off-by: Sunny --- tftestenv/flags.go | 4 +++ tftestenv/testenv.go | 86 +++++++++++++++++++++++++++++++++++--------- 2 files changed, 73 insertions(+), 17 deletions(-) diff --git a/tftestenv/flags.go b/tftestenv/flags.go index 7ce70c8..0273923 100644 --- a/tftestenv/flags.go +++ b/tftestenv/flags.go @@ -31,6 +31,9 @@ type Options struct { Existing bool // Verbose flag to enable output of terraform execution. Verbose bool + // DestroyOnly can be used to run the testenv in destroy only mode to + // perform cleanup. + DestroyOnly bool } var supportedProviders = []string{"aws", "azure", "gcp"} @@ -41,6 +44,7 @@ func (o *Options) Bindflags(fs *flag.FlagSet) { fs.BoolVar(&o.Retain, "retain", false, "retain the infrastructure for debugging purposes") fs.BoolVar(&o.Existing, "existing", false, "use existing infrastructure state for debugging purposes") fs.BoolVar(&o.Verbose, "verbose", false, "verbose output of the environment setup") + fs.BoolVar(&o.DestroyOnly, "destroy-only", false, "run in destroy-only mode and delete any existing infrastructure") } // Validate method ensures that the provider is set to one of the supported ones - aws, azure or gcp. diff --git a/tftestenv/testenv.go b/tftestenv/testenv.go index 0bd30ba..7d258e0 100644 --- a/tftestenv/testenv.go +++ b/tftestenv/testenv.go @@ -136,23 +136,7 @@ func New(ctx context.Context, scheme *runtime.Scheme, terraformPath string, kube return env, fmt.Errorf("failed to create build directory: %w", err) } - // Find or download terraform binary. - i := install.NewInstaller() - execPath, err := i.Ensure(ctx, []src.Source{ - &fs.AnyVersion{ - Product: &product.Terraform, - }, - &releases.LatestVersion{ - Product: product.Terraform, - InstallDir: buildDir, - }, - }) - if err != nil { - return env, fmt.Errorf("terraform exec path not found: %w", err) - } - log.Println("Terraform binary: ", execPath) - - env.tf, err = tfexec.NewTerraform(terraformPath, execPath) + env.tf, err = setUpTerraform(ctx, terraformPath, buildDir) if err != nil { return env, fmt.Errorf("could not create terraform instance: %w", err) } @@ -212,6 +196,28 @@ func New(ctx context.Context, scheme *runtime.Scheme, terraformPath string, kube return env, nil } +// setUpTerraform finds or downloads terraform binary and returns Terraform +// which can be used to run terraform operations. +func setUpTerraform(ctx context.Context, terraformPath string, buildDir string) (*tfexec.Terraform, error) { + // Find or download terraform binary. + i := install.NewInstaller() + execPath, err := i.Ensure(ctx, []src.Source{ + &fs.AnyVersion{ + Product: &product.Terraform, + }, + &releases.LatestVersion{ + Product: product.Terraform, + InstallDir: buildDir, + }, + }) + if err != nil { + return nil, fmt.Errorf("terraform exec path not found: %w", err) + } + log.Println("Terraform binary: ", execPath) + + return tfexec.NewTerraform(terraformPath, execPath) +} + // createAndConfigure creates the resources and configures the Environment with // the created resource. func (env *Environment) createAndConfigure(ctx context.Context, scheme *runtime.Scheme, kubeconfigPath string) error { @@ -262,3 +268,49 @@ func (env *Environment) StateOutput(ctx context.Context) (map[string]*tfjson.Sta } return state.Values.Outputs, nil } + +// Destroy configures a new Environment with the given configurations for +// terraform and runs terraform destroy. Ideally, this need not be used as the +// testenv New() handles graceful cleanup when shutdown signals are received. +// But in case the whole process gets terminated, use this to just perform a +// destroy of any created infrastructure. +// This can be used as the last step in CI to always run irrespective of success +// or failure of the test run to make sure the test infrastructure is destroyed. +// One such scenario is when the cloud provider takes longer than the usual time +// to provision the infrastructure and the test binary execution reaches timeout +// and the whole process gets terminated. This can be run in a separate step in +// CI to destroy the infrastructure. +func Destroy(ctx context.Context, terraformPath string, opts ...EnvironmentOption) error { + // Set a default logger if not set already. + runtimeLog.SetLogger(klogr.New()) + + env := &Environment{ + buildDir: "build", // Default build dir. + } + + // Process the options. + for _, opt := range opts { + opt(env) + } + + // Assume that the initial test run created the build directory. + cwd, err := os.Getwd() + if err != nil { + return fmt.Errorf("failed to get the current working directory: %w", err) + } + buildDir := filepath.Join(cwd, env.buildDir) + + env.tf, err = setUpTerraform(ctx, terraformPath, buildDir) + if err != nil { + return fmt.Errorf("could not create terraform instance: %w", err) + } + + if env.verbose { + env.tf.SetStdout(os.Stdout) + env.tf.SetStderr(os.Stderr) + } + + log.Println("Terraform destroy...") + // Bypass the terraform state lock. + return env.tf.Destroy(ctx, tfexec.Lock(false)) +} From dcc55c8801bdbe30b41c84bec44ce23dc4be83d1 Mon Sep 17 00:00:00 2001 From: Sunny Date: Mon, 8 Jan 2024 13:49:20 +0000 Subject: [PATCH 2/2] testenv: Introduce tf apply and destroy options The terraform apply and destroy operations can now be configured with testenv EnvironmentOption. These can be used to configure the behavior of apply and destroy. Signed-off-by: Sunny --- tftestenv/testenv.go | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/tftestenv/testenv.go b/tftestenv/testenv.go index 7d258e0..9e2ce35 100644 --- a/tftestenv/testenv.go +++ b/tftestenv/testenv.go @@ -57,6 +57,12 @@ type Environment struct { existing bool verbose bool buildDir string + // tfApplyOptions are the terraform apply options to use when running + // terraform apply. + tfApplyOptions []tfexec.ApplyOption + // tfDestroyOptions are the terraform destroy options to use when running + // terraform destroy. + tfDestroyOptions []tfexec.DestroyOption } // createKubeconfig create a kubeconfig for the target cluster and writes to @@ -106,6 +112,20 @@ func WithBuildDir(dir string) EnvironmentOption { } } +// WithTfApplyOptions configures terraform apply options. +func WithTfApplyOptions(opts ...tfexec.ApplyOption) EnvironmentOption { + return func(e *Environment) { + e.tfApplyOptions = append(e.tfApplyOptions, opts...) + } +} + +// WithTfDestroyOptions configures terraform destroy options. +func WithTfDestroyOptions(opts ...tfexec.DestroyOption) EnvironmentOption { + return func(e *Environment) { + e.tfDestroyOptions = append(e.tfDestroyOptions, opts...) + } +} + // New finds or downloads terraform binary, uses it to run terraform in the // given terraformPath to create a kubernetes cluster. A kubeconfig of the // created cluster is constructed at the given kubeconfigPath which is then used @@ -223,7 +243,7 @@ func setUpTerraform(ctx context.Context, terraformPath string, buildDir string) func (env *Environment) createAndConfigure(ctx context.Context, scheme *runtime.Scheme, kubeconfigPath string) error { // Apply Terraform, read the output values and construct kubeconfig. log.Println("Applying Terraform") - err := env.tf.Apply(ctx) + err := env.tf.Apply(ctx, env.tfApplyOptions...) if err != nil { return fmt.Errorf("error running apply: %v", err) } @@ -253,7 +273,7 @@ func (env *Environment) createAndConfigure(ctx context.Context, scheme *runtime. func (env *Environment) Stop(ctx context.Context) error { if !env.retain { log.Println("Destroying environment...") - if ferr := env.tf.Destroy(ctx); ferr != nil { + if ferr := env.tf.Destroy(ctx, env.tfDestroyOptions...); ferr != nil { return fmt.Errorf("could not destroy infrastructure: %w", ferr) } } @@ -311,6 +331,5 @@ func Destroy(ctx context.Context, terraformPath string, opts ...EnvironmentOptio } log.Println("Terraform destroy...") - // Bypass the terraform state lock. - return env.tf.Destroy(ctx, tfexec.Lock(false)) + return env.tf.Destroy(ctx, env.tfDestroyOptions...) }