Use custom script to find and delete AWS CCM created resources

Signed-off-by: Kyle Squizzato <[email protected]>

Makefile

@@ -306,6 +306,7 @@ dev-aws-destroy: ## Delete the AWS deployment
 
 .PHONY: dev-aws-nuke
 dev-aws-nuke: ## Warning: Destructive! Nuke all AWS resources deployed by 'dev-aws-apply', prefix with CLUSTER_NAME to nuke a specific cluster.
+	@CLUSTER_NAME=$(CLUSTER_NAME) YQ=$(YQ) bash -c ./scripts/aws-nuke-ccm.sh
 	@CLUSTER_NAME=$(CLUSTER_NAME) envsubst < config/dev/cloud_nuke.yaml.tpl > config/dev/cloud_nuke.yaml
 	DISABLE_TELEMETRY=true $(CLOUDNUKE) aws --region $$AWS_REGION --force --config config/dev/cloud_nuke.yaml --resource-type vpc,eip,nat-gateway,ec2-subnet,elb,elbv2,internet-gateway,network-interface,security-group
 	@rm config/dev/cloud_nuke.yaml

scripts/aws-nuke-ccm.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+# This script will remove all resources affiliated with the AWS CCM, such as
+# ELB or CSI driver resources that can not be filtered by cloud-nuke.
+# It should be ran after running cloud-nuke to remove any remaining resources.
+if [ -z $CLUSTER_NAME ]; then
+    echo "CLUSTER_NAME must be set"
+    exit 1
+fi
+
+if [ -z $YQ ]; then
+    YQ=$(which yq)
+fi
+
+echo "Checking for ELB with 'kubernetes.io/cluster/$CLUSTER_NAME' tag"
+for LOADBALANCER in $(aws elb describe-load-balancers --output yaml | yq '.LoadBalancerDescriptions[].LoadBalancerName');
+do
+    echo "Checking ELB: $LOADBALANCER for 'kubernetes.io/cluster/$CLUSTER_NAME tag"
+    DESCRIBE_TAGS=$(aws elb describe-tags \
+        --load-balancer-names $LOADBALANCER \
+        --output yaml | yq '.TagDescriptions[].Tags.[]' | grep 'kubernetes.io/cluster/$CLUSTER_NAME')
+    if [ ! -z "${DESCRIBE_TAGS}" ]; then
+        echo "Deleting ELB: $LOADBALANCER"
+        aws elb delete-load-balancer --load-balancer-name $LOADBALANCER
+    fi
+done
+
+echo "Checking for EBS Volumes with $CLUSTER_NAME within the 'kubernetes.io/created-for/pvc/name' tag"
+for VOLUME in $(aws ec2 describe-volumes --output yaml | yq '.Volumes[].VolumeId');
+do
+    echo "Checking EBS Volume: $VOLUME for $CLUSTER_NAME claim"
+    DESCRIBE_VOLUMES=$(aws ec2 describe-volumes \
+        --volume-id $VOLUME \
+        --output yaml | yq '.Volumes | to_entries[] | .value.Tags[] | select(.Key == "kubernetes.io/created-for/pvc/name")' | grep $CLUSTER_NAME)
+    if [ ! -z "${DESCRIBE_VOLUMES}" ]; then
+        echo "Deleting EBS Volume: $VOLUME"
+        aws ec2 delete-volume --volume-id $VOLUME
+    fi
+done