From 4c7cd32516f1c73584846dca02254e5d7eebde23 Mon Sep 17 00:00:00 2001
From: David Cheung <cheungdavid@google.com>
Date: Tue, 10 Oct 2023 14:58:06 +0000
Subject: [PATCH] Add wait_for_glbc_deletion for ingress resources cleanup.

* Add wait_for_glbc_deletion to wait for all ingress resources cleaned
  up before deleting cluster.
---
 Makefile       |  4 +--
 test/helper.sh | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 75 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 6e9eb32e..31800a65 100644
--- a/Makefile
+++ b/Makefile
@@ -29,8 +29,8 @@ test: bin/recipes-test
 	bin/recipes-test \
 		--run-in-prow=$(RUN_IN_PROW) \
 		--boskos-resource-type=$(BOSKOS_RESOURCE_TYPE) \
-		-test.v
-		-test.timeout=0
+		-test.v \
+		-test.timeout=180m
 
 .PHONY: clean
 clean:
diff --git a/test/helper.sh b/test/helper.sh
index 1f8a658c..2de9acdd 100644
--- a/test/helper.sh
+++ b/test/helper.sh
@@ -98,6 +98,77 @@ wait_for_managed_cert() {
     return 1
 }
 
+# Wait for the given ingress to completely delete its resources.
+# Arguments:
+#   Name of the ingress.
+#   Namespace of the ingress.
+# Returns:
+#   0 if resources are deleted within 1 hour, 1 if not.
+wait_for_glbc_deletion() {
+    local ingress_name test_name resource_suffix network ns ing_resource_name 
+    ingress_name="$1"
+    test_name="$2"
+    resource_suffix=$(get_hash "${test_name}")
+    network="gke-net-recipes-${resource_suffix}"
+
+    ns="${test_name}"
+    ing_resource_name="${ns}-${ingress_name}"
+    local ATTEMPT
+    for ATTEMPT in $(seq 360); do
+        local fwr thp thsp um bs neg
+        fwr=( $(gcloud compute forwarding-rules list \
+                --filter="NAME ~ ${ing_resource_name}" \
+                --format="value(NAME)") )
+        if  [[ "${#fwr[@]}" -ne 0 ]]; then
+            sleep 10
+            continue
+        fi
+
+        thp=( $(gcloud compute target-http-proxies list \
+                --filter="NAME ~ ${ing_resource_name}" \
+                --format="value(NAME)") )
+        if  [[ "${#thp[@]}" -ne 0 ]]; then
+            sleep 10
+            continue
+        fi
+
+        thsp=( $(gcloud compute target-https-proxies list \
+                --filter="NAME ~ ${ing_resource_name}" \
+                --format="value(NAME)") )
+        if  [[ "${#thsp[@]}" -ne 0 ]]; then
+            sleep 10
+            continue
+        fi
+
+        um=( $(gcloud compute url-maps list \
+                --filter="NAME ~ ${ing_resource_name}" \
+                --format="value(NAME)") )
+        if  [[ "${#um[@]}" -ne 0 ]]; then
+            sleep 10
+            continue
+        fi
+
+        bs=( $(gcloud compute backend-services list \
+                --filter="NAME ~ ${ing_resource_name}" \
+                --format="value(NAME)") )
+        if  [[ "${#bs[@]}" -ne 0 ]]; then
+            sleep 10
+            continue
+        fi
+
+        neg=( $(gcloud compute network-endpoint-groups list \
+                --filter="network ~ ${network}" \
+                --format="value(NAME)") )
+        if  [[ "${#neg[@]}" -ne 0 ]]; then
+            sleep 10
+            continue
+        fi
+        
+        return 0
+    done
+    return 1
+}
+
 # Basic setup that works for most recipe tests.
 # Create a network, and a subnet in the provided region.
 # Create an instance and a cluster in the given zone with the network and 
@@ -163,6 +234,8 @@ cleanup_gke_basic() {
         --zone="${zone}" --quiet || true
     gcloud compute networks subnets delete "${subnet}" \
         --region="${subnet_region}" --quiet || true
+    gcloud compute networks subnets delete "proxy-only-${resource_suffix}" \
+        --region="${subnet_region}" --quiet || true
 
     local firewalls fw
     # Cleanup the firewalls associated with the network before deleting it.