diff --git a/cloud-controller-manager/listeners.go b/cloud-controller-manager/listeners.go
index 9da17b15c..459868640 100644
--- a/cloud-controller-manager/listeners.go
+++ b/cloud-controller-manager/listeners.go
@@ -373,19 +373,23 @@ func EnsureListenersDeleted(
 
 	for _, loc := range local {
 		for _, rem := range remote {
-			if !isManagedByMyService(service, rem) {
-				continue
-			}
-			hasUserNode, err := rem.listenerHasUserManagedNode(ctx)
-			if err != nil {
-				return fmt.Errorf("check if listener has user managed node, error: %s", err.Error())
-			}
-			if hasUserNode {
-				klog.Infof("%s port %d vgroup has user managed node, skip", rem.NamedKey, rem.Port)
-				continue
-			}
+
 			if loc.Port == rem.Port {
-				err := loc.Remove(ctx)
+
+				if !isManagedByMyService(service, rem) {
+					continue
+				}
+
+				hasUserNode, err := rem.listenerHasUserManagedNode(ctx)
+				if err != nil {
+					return fmt.Errorf("check if listener has user managed node, error: %s", err.Error())
+				}
+				if hasUserNode {
+					klog.Infof("%s port %d vgroup has user managed node, skip", rem.NamedKey, rem.Port)
+					continue
+				}
+
+				err = loc.Remove(ctx)
 				if err != nil {
 					return fmt.Errorf("ensure listener: %s", err.Error())
 				}
@@ -1499,6 +1503,10 @@ func (n *Listener) listenerHasUserManagedNode(ctx context.Context) (bool, error)
 		return false, err
 	}
 
+	if n.VServerGroupId == "" {
+		return false, nil
+	}
+
 	remoteVg, err := n.Client.DescribeVServerGroupAttribute(
 		ctx,
 		&slb.DescribeVServerGroupAttributeArgs{