diff --git a/.github/workflows/integration-azure.yaml b/.github/workflows/integration-azure.yaml index 20cdad17..de2582ed 100644 --- a/.github/workflows/integration-azure.yaml +++ b/.github/workflows/integration-azure.yaml @@ -57,3 +57,15 @@ jobs: TF_VAR_enable_wi: ${{ matrix.enable-workload-id }} TF_VAR_wi_k8s_sa_name: test-workload-id TF_VAR_wi_k8s_sa_ns: default + - name: Ensure resource cleanup + if: ${{ always() }} + run: . .env && make destroy-azure + env: + ARM_CLIENT_ID: ${{ secrets.OCI_E2E_AZ_ARM_CLIENT_ID }} + ARM_CLIENT_SECRET: ${{ secrets.OCI_E2E_AZ_ARM_CLIENT_SECRET }} + ARM_SUBSCRIPTION_ID: ${{ secrets.OCI_E2E_AZ_ARM_SUBSCRIPTION_ID }} + ARM_TENANT_ID: ${{ secrets.OCI_E2E_AZ_ARM_TENANT_ID }} + TF_VAR_azure_location: ${{ vars.TF_VAR_azure_location }} + TF_VAR_enable_wi: ${{ matrix.enable-workload-id }} + TF_VAR_wi_k8s_sa_name: test-workload-id + TF_VAR_wi_k8s_sa_ns: default diff --git a/.github/workflows/integration-gcp.yaml b/.github/workflows/integration-gcp.yaml index d621bd75..156bcf8d 100644 --- a/.github/workflows/integration-gcp.yaml +++ b/.github/workflows/integration-gcp.yaml @@ -71,3 +71,13 @@ jobs: TF_VAR_enable_wi: ${{ matrix.enable-workload-id }} TF_VAR_wi_k8s_sa_name: test-workload-id TF_VAR_wi_k8s_sa_ns: default + - name: Ensure resource cleanup + if: ${{ always() }} + run: . .env && make destroy-gcp + env: + TF_VAR_gcp_project_id: ${{ vars.TF_VAR_gcp_project_id }} + TF_VAR_gcp_region: ${{ vars.TF_VAR_gcp_region }} + TF_VAR_gcp_zone: ${{ vars.TF_VAR_gcp_zone }} + TF_VAR_enable_wi: ${{ matrix.enable-workload-id }} + TF_VAR_wi_k8s_sa_name: test-workload-id + TF_VAR_wi_k8s_sa_ns: default diff --git a/oci/tests/integration/Makefile b/oci/tests/integration/Makefile index 33acd06f..2183bfd6 100644 --- a/oci/tests/integration/Makefile +++ b/oci/tests/integration/Makefile @@ -25,3 +25,15 @@ test-azure: test-gcp: $(MAKE) test PROVIDER_ARG="-provider gcp" + +destroy: + go test -timeout $(TEST_TIMEOUT) -v ./ $(GO_TEST_ARGS) $(PROVIDER_ARG) -destroy-only --tags=integration + +destroy-aws: + $(MAKE) destroy PROVIDER_ARG="-provider aws" + +destroy-azure: + $(MAKE) destroy PROVIDER_ARG="-provider azure" + +destroy-gcp: + $(MAKE) destroy PROVIDER_ARG="-provider gcp" diff --git a/oci/tests/integration/README.md b/oci/tests/integration/README.md index 93b47d21..3fc14ce3 100644 --- a/oci/tests/integration/README.md +++ b/oci/tests/integration/README.md @@ -79,6 +79,10 @@ variables using use the terraform configuration below. Please make sure all the requirements of azure-gh-actions are followed before running it. +**NOTE:** When running the following for a repo under an organization, set the +environment variable `GITHUB_ORGANIZATION` if setting the `owner` in the +`github` provider doesn't work. + ```hcl provider "github" { owner = "fluxcd" @@ -189,6 +193,10 @@ variables using use the terraform configuration below. Please make sure all the requirements of gcp-gh-actions are followed before running it. +**NOTE:** When running the following for a repo under an organization, set the +environment variable `GITHUB_ORGANIZATION` if setting the `owner` in the +`github` provider doesn't work. + ```hcl provider "google" {} @@ -244,6 +252,14 @@ TEST_IMG=fluxcd/testapp:test go test -timeout 30m -v ./ -verbose -retain -provid ... ``` +If not configured explicitly to retain the infrastructure, at the end of the +test, the test infrastructure is deleted. In case of any failure due to which +the resources don't get deleted, the `make destroy-*` commands can be run for +the respective provider. This will run terraform destroy in the respective +provider's terraform configuration directory. This can be used to quickly +destroy the infrastructure without going through the provision-test-destroy +steps. + ## Workload Identity By default, the tests use node identity for authentication. To run the integration tests on clusters with workload identity diff --git a/oci/tests/integration/go.mod b/oci/tests/integration/go.mod index a5b10f82..6b760823 100644 --- a/oci/tests/integration/go.mod +++ b/oci/tests/integration/go.mod @@ -6,8 +6,9 @@ replace github.com/fluxcd/pkg/oci => ../../ require ( github.com/fluxcd/pkg/oci v0.32.0 - github.com/fluxcd/test-infra/tftestenv v0.0.0-20230720084205-d40ee5473f22 + github.com/fluxcd/test-infra/tftestenv v0.0.0-20240108135005-b58e0c4e0cfa github.com/google/go-containerregistry v0.17.0 + github.com/hashicorp/terraform-exec v0.18.1 github.com/hashicorp/terraform-json v0.17.1 github.com/onsi/gomega v1.30.0 k8s.io/api v0.28.4 @@ -65,7 +66,6 @@ require ( github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/hc-install v0.5.1 // indirect - github.com/hashicorp/terraform-exec v0.18.1 // indirect github.com/imdario/mergo v0.3.15 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect diff --git a/oci/tests/integration/go.sum b/oci/tests/integration/go.sum index 36493bfd..ff8e08c7 100644 --- a/oci/tests/integration/go.sum +++ b/oci/tests/integration/go.sum @@ -71,8 +71,8 @@ github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= github.com/evanphx/json-patch/v5 v5.7.0 h1:nJqP7uwL84RJInrohHfW0Fx3awjbm8qZeFv0nW9SYGc= github.com/evanphx/json-patch/v5 v5.7.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= -github.com/fluxcd/test-infra/tftestenv v0.0.0-20230720084205-d40ee5473f22 h1:1f0EQM0kPX2px9FanVMyD/UrHFQ9zqFg58M42Y8bBts= -github.com/fluxcd/test-infra/tftestenv v0.0.0-20230720084205-d40ee5473f22/go.mod h1:liFlLEXgambGVdWSJ4JzbIHf1Vjpp1HwUyPazPIVZug= +github.com/fluxcd/test-infra/tftestenv v0.0.0-20240108135005-b58e0c4e0cfa h1:JdI+rVwGF5gBYt+UBijOVzXtq7aAU80vgksMNXSCCfU= +github.com/fluxcd/test-infra/tftestenv v0.0.0-20240108135005-b58e0c4e0cfa/go.mod h1:liFlLEXgambGVdWSJ4JzbIHf1Vjpp1HwUyPazPIVZug= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= diff --git a/oci/tests/integration/suite_test.go b/oci/tests/integration/suite_test.go index 09841194..2ddc566e 100644 --- a/oci/tests/integration/suite_test.go +++ b/oci/tests/integration/suite_test.go @@ -29,6 +29,7 @@ import ( "testing" "time" + "github.com/hashicorp/terraform-exec/tfexec" tfjson "github.com/hashicorp/terraform-json" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -80,6 +81,9 @@ var ( // verbose flag to enable output of terraform execution. verbose = flag.Bool("verbose", false, "verbose output of the environment setup") + // destroyOnly flag to destroy any provisioned infrastructure. + destroyOnly = flag.Bool("destroy-only", false, "run in destroy-only mode and delete any existing infrastructure") + // testRepos is a map of registry common name and URL of the test // repositories. This is used as the test cases to run the tests against. // The registry common name need not be the actual registry address but an @@ -153,15 +157,6 @@ func TestMain(m *testing.M) { flag.Parse() ctx := context.TODO() - appImg := os.Getenv("TEST_IMG") - if appImg == "" { - log.Fatal("TEST_IMG must be set to the test application image, cannot be empty") - } - - localImgs := map[string]string{ - "app": appImg, - } - // Validate the provider. if *targetProvider == "" { log.Fatalf("-provider flag must be set to one of %v", supportedProviders) @@ -181,6 +176,30 @@ func TestMain(m *testing.M) { log.Fatalf("Failed to get provider config for %q", *targetProvider) } + // Run destroy-only mode if enabled. + if *destroyOnly { + log.Println("Running in destroy-only mode...") + envOpts := []tftestenv.EnvironmentOption{ + tftestenv.WithVerbose(*verbose), + // Ignore any state lock in destroy-only mode. + tftestenv.WithTfDestroyOptions(tfexec.Lock(false)), + } + if err := tftestenv.Destroy(ctx, providerCfg.terraformPath, envOpts...); err != nil { + panic(err) + } + os.Exit(0) + } + + // Check the test app image. + appImg := os.Getenv("TEST_IMG") + if appImg == "" { + log.Fatal("TEST_IMG must be set to the test application image, cannot be empty") + } + + localImgs := map[string]string{ + "app": appImg, + } + // Construct scheme to be added to the kubeclient. scheme := runtime.NewScheme()