From 465079af938e1a48f177ac85e133bb31734b160d Mon Sep 17 00:00:00 2001
From: Xie Zheng <xizheng@vmware.com>
Date: Thu, 9 May 2024 16:47:59 +0800
Subject: [PATCH] Add clean avi subnet ports before deleting vpc

Avi controller will create subnetport on avi subnet of VPC, when there are stale subnetports left behind, it will fail to cleanup VPC complaining "IpAddressSubnet cannot be deleted as IP from subnet range is being used.", if delete avi subnet ports before deleting VPC, it will succeed to cleanup vpc even though there is avi subnet.

Signed-off-by: Xie Zheng <xizheng@vmware.com>
---
 pkg/nsx/services/vpc/clean_avi.go | 58 +++++++++++++++++++++++++++++++
 pkg/nsx/services/vpc/vpc.go       |  4 +++
 2 files changed, 62 insertions(+)
 create mode 100644 pkg/nsx/services/vpc/clean_avi.go

diff --git a/pkg/nsx/services/vpc/clean_avi.go b/pkg/nsx/services/vpc/clean_avi.go
new file mode 100644
index 000000000..c62a0b87a
--- /dev/null
+++ b/pkg/nsx/services/vpc/clean_avi.go
@@ -0,0 +1,58 @@
+package vpc
+
+import (
+	"context"
+	"errors"
+
+	mapset "github.com/deckarep/golang-set"
+
+	"github.com/vmware-tanzu/nsx-operator/pkg/nsx"
+	nsxutil "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util"
+)
+
+type (
+	mapInterface = map[string]interface{}
+)
+
+const (
+	PolicyAPI                = "policy/api/v1"
+	AviSubnetPortsPathSuffix = "/subnets/_AVI_SUBNET--LB/ports/"
+)
+
+func httpGetAviPortsPaths(cluster *nsx.Cluster, vpcPath string) (mapset.Set, error) {
+	aviSubnetPortsPath := vpcPath + AviSubnetPortsPathSuffix
+	url := PolicyAPI + aviSubnetPortsPath
+
+	resp, err := cluster.HttpGet(url)
+	if err != nil {
+		return nil, err
+	}
+	aviPathSet := mapset.NewSet()
+	for _, item := range resp["results"].([]interface{}) {
+		aviPathSet.Add(item.(mapInterface)["path"].(string))
+	}
+	return aviPathSet, nil
+}
+
+func CleanAviSubnetPorts(ctx context.Context, cluster *nsx.Cluster, vpcPath string) error {
+	log.Info("Deleting Avi subnetports started")
+
+	allPaths, err := httpGetAviPortsPaths(cluster, vpcPath)
+	if err != nil {
+		return err
+	}
+
+	log.Info("Deleting Avi subnetport", "paths", allPaths)
+	for _, path := range allPaths.ToSlice() {
+		url := PolicyAPI + path.(string)
+		select {
+		case <-ctx.Done():
+			return errors.Join(nsxutil.TimeoutFailed, ctx.Err())
+		default:
+			if err := cluster.HttpDelete(url); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
diff --git a/pkg/nsx/services/vpc/vpc.go b/pkg/nsx/services/vpc/vpc.go
index 478b9d68b..b6ce45fcf 100644
--- a/pkg/nsx/services/vpc/vpc.go
+++ b/pkg/nsx/services/vpc/vpc.go
@@ -618,6 +618,10 @@ func (s *VPCService) Cleanup(ctx context.Context) error {
 		case <-ctx.Done():
 			return errors.Join(nsxutil.TimeoutFailed, ctx.Err())
 		default:
+			// first clean avi subnet ports, or else vpc delete will fail
+			if err := CleanAviSubnetPorts(ctx, s.NSXClient.Cluster, *vpc.Path); err != nil {
+				return err
+			}
 			if err := s.DeleteVPC(*vpc.Path); err != nil {
 				return err
 			}