tests/int: Add separate resource cleanup step

Introduce a destroy-only mode in the test runner to run terraform
destroy for the respective cloud provider configurations. This can be
used to destroy cloud resources without going through the whole
provision-test process.

Add a new step in github actions workflow to run the test binary in
destoy-only mode at the very end irrespective of the result of the
previous steps. This ensures that the infrastructure is always
destroyed, even if the CI job is cancelled.

Signed-off-by: Sunny <[email protected]>

.github/workflows/e2e-azure.yaml

@@ -123,3 +123,14 @@ jobs:
           echo $GITREPO_SSH_PUB_CONTENTS | base64 -d > ./build/ssh/key.pub
           export GITREPO_SSH_PUB_PATH=build/ssh/key.pub
           make test-azure
+      - name: Ensure resource cleanup
+        if: ${{ always() }}
+        env:
+          ARM_CLIENT_ID: ${{ secrets.AZ_ARM_CLIENT_ID }}
+          ARM_CLIENT_SECRET: ${{ secrets.AZ_ARM_CLIENT_SECRET }}
+          ARM_SUBSCRIPTION_ID: ${{ secrets.AZ_ARM_SUBSCRIPTION_ID }}
+          ARM_TENANT_ID: ${{ secrets.AZ_ARM_TENANT_ID }}
+          TF_VAR_azuredevops_org: ${{ secrets.TF_VAR_azuredevops_org }}
+          TF_VAR_azuredevops_pat: ${{ secrets.TF_VAR_azuredevops_pat }}
+          TF_VAR_location: ${{ vars.TF_VAR_azure_location }}
+        run: source .env && make destroy-azure

.github/workflows/e2e-gcp.yaml

@@ -90,3 +90,13 @@ jobs:
           echo $GITREPO_SSH_PUB_CONTENTS | base64 -d > ./build/ssh/key.pub
           export GITREPO_SSH_PUB_PATH=build/ssh/key.pub
           make test-gcp
+      - name: Ensure resource cleanup
+        if: ${{ always() }}
+        env:
+          TF_VAR_gcp_project_id: ${{ vars.TF_VAR_gcp_project_id }}
+          TF_VAR_gcp_region: ${{ vars.TF_VAR_gcp_region }}
+          TF_VAR_gcp_zone: ${{ vars.TF_VAR_gcp_zone }}
+          TF_VAR_gcp_email: ${{ secrets.TF_VAR_gcp_email }}
+          TF_VAR_gcp_keyring: ${{ secrets.TF_VAR_gcp_keyring }}
+          TF_VAR_gcp_crypto_key: ${{ secrets.TF_VAR_gcp_crypto_key }}
+        run: source .env && make destroy-gcp

tests/integration/Makefile

@@ -17,6 +17,14 @@ test-azure:
 test-gcp:
 	$(MAKE) test PROVIDER_ARG="-provider gcp"
 
+destroy:
+	go test -timeout $(TEST_TIMEOUT) -v ./ $(GO_TEST_ARGS) $(PROVIDER_ARG) -destroy-only
+
+destroy-azure:
+	$(MAKE) destroy PROVIDER_ARG="-provider azure"
+
+destroy-gcp:
+	$(MAKE) destroy PROVIDER_ARG="-provider gcp"
 
 sops-check:
 ifeq ($(shell which sops),)

tests/integration/README.md

@@ -74,6 +74,10 @@ variables using
 use the terraform configuration below. Please make sure all the requirements of
 azure-gh-actions are followed before running it.
 
+**NOTE:** When running the following for a repo under an organization, set the
+environment variable `GITHUB_ORGANIZATION` if setting the `owner` in the
+`github` provider doesn't work.
+
 ```hcl
 provider "github" {
   owner = "fluxcd"
@@ -218,7 +222,7 @@ variables using
 use the terraform configuration below. Please make sure all the requirements of
 gcp-gh-actions are followed before running it.
 
-**NOTE:** When running the following for a repo under and organization, set the
+**NOTE:** When running the following for a repo under an organization, set the
 environment variable `GITHUB_ORGANIZATION` if setting the `owner` in the
 `github` provider doesn't work.
 
@@ -345,8 +349,13 @@ We also pull two version of `ghcr.io/stefanprodan/podinfo` image. These images a
 Container Registry and used to test `ImageRepository` and `ImageUpdateAutomation`. The terraform resources get created
 and the tests are run.
 
-**IMPORTANT:** In case the terraform infrastructure results in a bad state, maybe due to a crash during the apply,
-the whole infrastructure can be destroyed by running terraform destroy in terraform/<provider> directory.
+If not configured explicitly to retain the infrastructure, at the end of the
+test, the test infrastructure is deleted. In case of any failure due to which
+the resources don't get deleted, the `make destroy-*` commands can be run for
+the respective provider. This will run terraform destroy in the respective
+provider's terraform configuration directory. This can be used to quickly
+destroy the infrastructure without going through the provision-test-destroy
+steps.
 
 ### Debugging the tests
 

tests/integration/go.mod

@@ -17,9 +17,10 @@ require (
 	github.com/fluxcd/pkg/git/gogit v0.16.1
 	github.com/fluxcd/pkg/runtime v0.43.2
 	github.com/fluxcd/source-controller/api v1.2.3
-	github.com/fluxcd/test-infra/tftestenv v0.0.0-20230831142147-627bca8e7916
+	github.com/fluxcd/test-infra/tftestenv v0.0.0-20240108135005-b58e0c4e0cfa
 	github.com/go-git/go-git/v5 v5.11.0
 	github.com/google/go-containerregistry v0.16.1
+	github.com/hashicorp/terraform-exec v0.18.1
 	github.com/hashicorp/terraform-json v0.16.0
 	github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5
 	github.com/onsi/gomega v1.30.0
@@ -89,7 +90,6 @@ require (
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/hashicorp/hc-install v0.5.0 // indirect
-	github.com/hashicorp/terraform-exec v0.18.1 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/josharian/intern v1.0.0 // indirect

tests/integration/go.sum

@@ -153,8 +153,8 @@ github.com/fluxcd/pkg/version v0.2.2 h1:ZpVXECeLA5hIQMft11iLp6gN3cKcz6UNuVTQPw/b
 github.com/fluxcd/pkg/version v0.2.2/go.mod h1:NGnh/no8S6PyfCDxRFrPY3T5BUnqP48MxfxNRU0z8C0=
 github.com/fluxcd/source-controller/api v1.2.3 h1:71mXv3Qg9HEhcpqOq1ObmoE+P/HuZNaAvxfI7dqZMo8=
 github.com/fluxcd/source-controller/api v1.2.3/go.mod h1:5gaIVVH7hgb8p3HKFp8P6hGmZEC8fKSt4EcrG3g5vZI=
-github.com/fluxcd/test-infra/tftestenv v0.0.0-20230831142147-627bca8e7916 h1:w9UGknpfAGbiObQALZiuWYGeriAU1wKCFTmI2tj/96M=
-github.com/fluxcd/test-infra/tftestenv v0.0.0-20230831142147-627bca8e7916/go.mod h1:liFlLEXgambGVdWSJ4JzbIHf1Vjpp1HwUyPazPIVZug=
+github.com/fluxcd/test-infra/tftestenv v0.0.0-20240108135005-b58e0c4e0cfa h1:JdI+rVwGF5gBYt+UBijOVzXtq7aAU80vgksMNXSCCfU=
+github.com/fluxcd/test-infra/tftestenv v0.0.0-20240108135005-b58e0c4e0cfa/go.mod h1:liFlLEXgambGVdWSJ4JzbIHf1Vjpp1HwUyPazPIVZug=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=

tests/integration/suite_test.go

@@ -26,6 +26,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/hashicorp/terraform-exec/tfexec"
 	tfjson "github.com/hashicorp/terraform-json"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/client-go/kubernetes/scheme"
@@ -197,6 +198,20 @@ func TestMain(m *testing.M) {
 		log.Fatalf("Failed to get provider config for %q", infraOpts.Provider)
 	}
 
+	// Run destroy-only mode if enabled.
+	if infraOpts.DestroyOnly {
+		log.Println("Running in destroy-only mode...")
+		envOpts := []tftestenv.EnvironmentOption{
+			tftestenv.WithVerbose(infraOpts.Verbose),
+			// Ignore any state lock in destroy-only mode.
+			tftestenv.WithTfDestroyOptions(tfexec.Lock(false)),
+		}
+		if err := tftestenv.Destroy(ctx, providerCfg.terraformPath, envOpts...); err != nil {
+			panic(err)
+		}
+		os.Exit(0)
+	}
+
 	// Initialize with non-zero exit code to indicate failure by default unless
 	// set by a successful test run.
 	exitCode := 1