From ed5c462311638ba8a4bf63a806ca70bae4918b98 Mon Sep 17 00:00:00 2001 From: Kyle Squizzato Date: Wed, 21 Aug 2024 17:02:27 -0700 Subject: [PATCH] Add 'make dev-aws-nuke' target, add support for nuking AWS resources in test Instead of using cloud-nuke as a library, add a cloud-nuke Makefile target so users can take advantage of cloud-nuke filtering for dev work outside of test. Signed-off-by: Kyle Squizzato --- Makefile | 21 +- config/dev/cloud_nuke.yaml.tpl | 340 +++++++++++++++++++++++++++++++++ config/dev/deployment.yaml | 14 +- test/e2e/e2e_test.go | 3 + test/kubeclient/kubeclient.go | 14 ++ test/utils/deployment.go | 7 + 6 files changed, 389 insertions(+), 10 deletions(-) create mode 100644 config/dev/cloud_nuke.yaml.tpl diff --git a/Makefile b/Makefile index 815e303e0..622d0f575 100644 --- a/Makefile +++ b/Makefile @@ -191,6 +191,7 @@ KIND_NETWORK ?= kind REGISTRY_NAME ?= hmc-local-registry REGISTRY_PORT ?= 5001 REGISTRY_REPO ?= oci://127.0.0.1:$(REGISTRY_PORT)/charts +CLUSTER_NAME ?= $(shell $(YQ) '.metadata.name' ./config/dev/deployment.yaml) AWS_CREDENTIALS=${AWS_B64ENCODED_CREDENTIALS} @@ -275,15 +276,21 @@ dev-apply: kind-deploy registry-deploy dev-push dev-deploy dev-templates dev-aws dev-destroy: kind-undeploy registry-undeploy .PHONY: dev-aws-apply -dev-aws-apply: +dev-aws-apply: ## Apply the AWS deployment $(KUBECTL) -n $(NAMESPACE) apply -f config/dev/deployment.yaml .PHONY: dev-aws-destroy -dev-aws-destroy: +dev-aws-destroy: ## Delete the AWS deployment $(KUBECTL) -n $(NAMESPACE) delete -f config/dev/deployment.yaml +.PHONY: dev-aws-nuke +dev-aws-nuke: ## Warning: Destructive! Nuke all AWS resources deployed by 'dev-aws-apply', prefix with CLUSTER_NAME to nuke a specific cluster. + @CLUSTER_NAME=$(CLUSTER_NAME) envsubst < config/dev/cloud_nuke.yaml.tpl > config/dev/cloud_nuke.yaml + $(CLOUDNUKE) aws --region us-west-2 --force --config config/dev/cloud_nuke.yaml --resource-type vpc,eip,nat-gateway,ec2-subnet,internet-gateway,network-interface,security-group + @rm config/dev/cloud_nuke.yaml + .PHONY: cli-install -cli-install: clusterawsadm clusterctl +cli-install: clusterawsadm clusterctl cloud-nuke ##@ Dependencies @@ -312,6 +319,7 @@ KIND ?= $(LOCALBIN)/kind-$(KIND_VERSION) YQ ?= $(LOCALBIN)/yq-$(YQ_VERSION) CLUSTERAWSADM ?= $(LOCALBIN)/clusterawsadm CLUSTERCTL ?= $(LOCALBIN)/clusterctl +CLOUDNUKE ?= $(LOCALBIN)/cloud-nuke ADDLICENSE ?= $(LOCALBIN)/addlicense-$(ADDLICENSE_VERSION) ## Tool Versions @@ -321,6 +329,7 @@ GOLANGCI_LINT_VERSION ?= v1.60.1 HELM_VERSION ?= v3.15.1 KIND_VERSION ?= v0.23.0 YQ_VERSION ?= v4.44.2 +CLOUDNUKE_VERSION = v0.37.1 CLUSTERAWSADM_VERSION ?= v2.5.2 CLUSTERCTL_VERSION ?= v1.7.3 ADDLICENSE_VERSION ?= v1.1.1 @@ -372,6 +381,12 @@ yq: $(YQ) ## Download yq locally if necessary. $(YQ): | $(LOCALBIN) $(call go-install-tool,$(YQ),github.com/mikefarah/yq/v4,${YQ_VERSION}) +.PHONY: cloud-nuke +cloud-nuke: $(CLOUDNUKE) ## Download cloud-nuke locally if necessary. +$(CLOUDNUKE): | $(LOCALBIN) + curl -sL https://github.com/gruntwork-io/cloud-nuke/releases/download/$(CLOUDNUKE_VERSION)/cloud-nuke_$(OS)_$(ARCH) -o $(CLOUDNUKE) + chmod +x $(CLOUDNUKE) + .PHONY: clusterawsadm clusterawsadm: $(CLUSTERAWSADM) ## Download clusterawsadm locally if necessary. $(CLUSTERAWSADM): | $(LOCALBIN) diff --git a/config/dev/cloud_nuke.yaml.tpl b/config/dev/cloud_nuke.yaml.tpl new file mode 100644 index 000000000..1888fe965 --- /dev/null +++ b/config/dev/cloud_nuke.yaml.tpl @@ -0,0 +1,340 @@ +# This config file is used by cloud-nuke to clean up named resources associated +# with a specific managed cluster across an AWS account. CLUSTER_NAME is +# typically the metadata.name of the Deployment. +# The resources listed here are ALL of the potential resources that can be +# filtered by cloud-nuke, except for IAM resources since we'll never touch those. +# See: https://github.com/gruntwork-io/cloud-nuke?tab=readme-ov-file#whats-supported +# +# Usage: +# - 'make aws-dev-nuke' will nuke resources affiliated with config/dev/deployment.yaml +# - 'CLUSTER_NAME=foo make aws-dev-nuke' will nuke resources affiliated with an AWS cluster named 'foo' +# Check cluster names with 'kubectl get clusters -n hmc-system' + +ACM: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +APIGateway: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +APIGatewayV2: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +AccessAnalyzer: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +AutoScalingGroup: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +AppRunnerService: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +BackupVault: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +CloudWatchAlarm: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +CloudWatchDashboard: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +CloudWatchLogGroup: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +CloudtrailTrail: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +CodeDeployApplications: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +ConfigServiceRecorder: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +ConfigServiceRule: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +DataSyncTask: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +DynamoDB: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +EBSVolume: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +ElasticBeanstalk: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +EC2: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +EC2DedicatedHosts: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +EC2KeyPairs: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +EC2IPAM: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +EC2IPAMPool: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +EC2IPAMResourceDiscovery: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +EC2IPAMScope: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +EC2PlacementGroups: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +EC2Subnet: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +EC2Endpoint: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +ECRRepository: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +ECSCluster: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +ECSService: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +EKSCluster: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +ELBv1: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +ELBv2: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +ElasticFileSystem: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +ElasticIP: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +Elasticache: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +ElasticacheParameterGroups: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +ElasticacheSubnetGroups: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +InternetGateway: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +EgressOnlyInternetGateway: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +LambdaFunction: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +LaunchConfiguration: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +LaunchTemplate: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +MSKCluster: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +NatGateway: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +NetworkACL: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +NetworkInterface: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +OIDCProvider: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +OpenSearchDomain: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +Redshift: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +DBClusters: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +DBInstances: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +RdsParameterGroup: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +DBSubnetGroups: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +RDSProxy: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +s3: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +s3AccessPoint: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +S3ObjectLambdaAccessPoint: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +S3MultiRegionAccessPoint: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +SecurityGroup: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +SesConfigurationset: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +SesEmailTemplates: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +SesIdentity: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +SesReceiptRuleSet: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +SesReceiptFilter: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +SNS: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +SQS: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +SageMakerNotebook: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +SecretsManager: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +VPC: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +Route53HostedZone: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +Route53CIDRCollection: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +Route53TrafficPolicy: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +NetworkFirewall: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +NetworkFirewallPolicy: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +NetworkFirewallRuleGroup: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +NetworkFirewallTLSConfig: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +NetworkFirewallResourcePolicy: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +VPCLatticeService: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +VPCLatticeServiceNetwork: + include: + names_regex: + - '^${CLUSTER_NAME}.*' +VPCLatticeTargetGroup: + include: + names_regex: + - '^${CLUSTER_NAME}.*' diff --git a/config/dev/deployment.yaml b/config/dev/deployment.yaml index 364521093..58ba18e32 100644 --- a/config/dev/deployment.yaml +++ b/config/dev/deployment.yaml @@ -3,15 +3,15 @@ kind: Deployment metadata: name: aws-dev spec: + template: aws-standalone-cp config: + region: us-east-2 + publicIP: true + controlPlaneNumber: 1 + workersNumber: 1 controlPlane: - amiID: ami-0989c067ff3da4b27 + amiID: ami-02f3416038bdb17fb instanceType: t3.small - controlPlaneNumber: 1 - publicIP: true - region: us-west-2 worker: - amiID: ami-0989c067ff3da4b27 + amiID: ami-02f3416038bdb17fb instanceType: t3.small - workersNumber: 1 - template: aws-standalone-cp diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 7aa7b81f5..13ed9be72 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -102,6 +102,9 @@ var _ = Describe("controller", Ordered, func() { AfterAll(func() { // Purge the AWS resources, the AfterAll for the controller will // clean up the management cluster. + cmd := exec.Command("make", "dev-aws-nuke") + _, err := utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred()) }) It("should work with an AWS provider", func() { diff --git a/test/kubeclient/kubeclient.go b/test/kubeclient/kubeclient.go index f10d4dd2c..f2a954dda 100644 --- a/test/kubeclient/kubeclient.go +++ b/test/kubeclient/kubeclient.go @@ -1,3 +1,17 @@ +// Copyright 2024 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package kubeclient import ( diff --git a/test/utils/deployment.go b/test/utils/deployment.go index e375dc5a9..875668479 100644 --- a/test/utils/deployment.go +++ b/test/utils/deployment.go @@ -66,6 +66,13 @@ func ConfigureDeploymentConfig(provider ProviderType, templateName Template) (st // Modify the existing ./config/dev/deployment.yaml file to use the // AMI we just found and our AWS_REGION. + if metadata, ok := deploymentConfig["metadata"].(map[string]interface{}); ok { + metadata["name"] = generatedName + } else { + // Ensure we always have a metadata.name field populated. + deploymentConfig["metadata"] = map[string]interface{}{"name": generatedName} + } + if spec, ok := deploymentConfig["spec"].(map[string]interface{}); ok { if config, ok := spec["config"].(map[string]interface{}); ok { if awsRegion != "" {