🌱 add: lb multiple subnet support, just check in a same vpc

vngcloud · Sep 18, 2024 · 5d47ed9 · 5d47ed9
1 parent ea32779
commit 5d47ed9
Show file tree

Hide file tree

Showing 4 changed files with 57 additions and 34 deletions.
diff --git a/pkg/ingress/controller/controller.go b/pkg/ingress/controller/controller.go
@@ -891,24 +891,21 @@ func (c *Controller) inspectIngress(ing *nwv1.Ingress) (*Expander, error) {
 		return nil, err
 	}
 
-	// Check the nodes are in the same subnet
-	subnetID, retErr := vngcloudutil.EnsureNodesInCluster(servers)
+	// Check the nodes are in the same VPC
+	networkID, subnetIDs, retErr := vngcloudutil.EnsureNodesInCluster(servers)
 	if retErr != nil {
-		klog.Errorf("All node are not in a same subnet: %v", retErr)
+		klog.Errorf("All node are not in a same VPC: %v", retErr)
 		return nil, retErr
 	}
-	networkID := vngcloudutil.GetNetworkID(servers, subnetID)
-	if networkID == "" {
-		klog.Errorf("Failed to get networkID from subnetID: %s", subnetID)
-		return nil, vErrors.ErrNetworkIDNotFound
-	}
-	subnet, err := vngcloudutil.GetSubnet(c.vServerSC, c.getProjectID(), networkID, subnetID)
+	subnet, err := vngcloudutil.GetSubnet(c.vServerSC, c.getProjectID(), networkID, subnetIDs[0])
 	if err != nil {
 		klog.Errorf("Failed to get subnet: %v", err)
 		return nil, err
 	}
+	ingressInspect.NetworkID = networkID
+	ingressInspect.SubnetID = subnetIDs[0]
 	ingressInspect.SubnetCIDR = subnet.CIDR
-	ingressInspect.LbOptions.SubnetID = subnetID
+	ingressInspect.LbOptions.SubnetID = subnetIDs[0]
 	ingressInspect.InstanceIDs = providerIDs
 
 	mapHostTLS := func(ing *nwv1.Ingress) map[string]bool {
@@ -1116,6 +1113,15 @@ func (c *Controller) ensureLoadBalancerInstance(inspect *Expander) (string, erro
 		klog.Errorf("error when get lb: %v", err)
 		return inspect.serviceConf.LoadBalancerID, err
 	}
+	if inspect.SubnetID != lb.SubnetID {
+		subnet, err := vngcloudutil.GetSubnet(c.vServerSC, c.getProjectID(), inspect.NetworkID, lb.SubnetID)
+		if err != nil {
+			klog.Errorf("Failed to get subnet: %v", err)
+			return "", err
+		}
+		inspect.SubnetID = lb.SubnetID
+		inspect.SubnetCIDR = subnet.CIDR
+	}
 
 	checkDetailLB := func() {
 		if lb.Name != inspect.serviceConf.LoadBalancerName {

diff --git a/pkg/utils/expander.go b/pkg/utils/expander.go
@@ -54,6 +54,8 @@ type IngressInspect struct {
 	Namespace   string
 	LbOptions   *loadbalancer.CreateOpts // create options for lb
 	SubnetCIDR  string
+	SubnetID    string // nodes may have different subnets, so we need to store the subnet id of LB
+	NetworkID   string // nodes may have different subnets, so we need to store the subnet id of LB
 
 	PolicyExpander       []*PolicyExpander
 	PoolExpander         []*PoolExpander

diff --git a/pkg/utils/vngcloud/utils.go b/pkg/utils/vngcloud/utils.go
@@ -54,26 +54,35 @@ func SetupPortalInfo(pProvider *client2.ProviderClient, pMedatata metadata.IMeta
 	}, nil
 }
 
-func EnsureNodesInCluster(pserver []*lObjects.Server) (string, error) {
-	subnetMapping := make(map[string]int)
+func getKeys(m map[string]bool) []string {
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	return keys
+}
+func EnsureNodesInCluster(pserver []*lObjects.Server) (string, []string, error) {
+	networkIDs := make(map[string]bool)
+	subnetIDs := make(map[string]bool)
+
+	if len(pserver) == 0 {
+		return "", []string{}, vErrors.ErrNodesAreNotSameSubnet
+	}
+
 	for _, server := range pserver {
-		subnets := listSubnetIDs(server)
-		for _, subnet := range subnets {
-			if smi, ok := subnetMapping[subnet]; !ok {
-				subnetMapping[subnet] = 1
-			} else {
-				subnetMapping[subnet] = smi + 1
-			}
+		for _, subnet := range server.InternalInterfaces {
+			networkIDs[subnet.NetworkUuid] = true
+			subnetIDs[subnet.SubnetUuid] = true
 		}
 	}
 
-	for subnet, count := range subnetMapping {
-		if count == len(pserver) && len(subnet) > 0 {
-			return subnet, nil
-		}
+	if len(networkIDs) != 1 || len(subnetIDs) < 1 {
+		return "", nil, vErrors.ErrNodesAreNotSameSubnet
 	}
 
-	return "", vErrors.ErrNodesAreNotSameSubnet
+	klog.Infof("EnsureNodesInCluster; networkIDs: %v, subnetIDs: %v", networkIDs, subnetIDs)
+
+	return getKeys(networkIDs)[0], getKeys(subnetIDs), nil
 }
 
 func listSubnetIDs(s *lObjects.Server) []string {

diff --git a/pkg/vngcloud/vlb.go b/pkg/vngcloud/vlb.go
@@ -519,24 +519,21 @@ func (c *vLB) inspectService(pService *lCoreV1.Service, pNodes []*lCoreV1.Node)
 		return nil, err
 	}
 
-	// Check the nodes are in the same subnet
-	subnetID, retErr := vngcloudutil.EnsureNodesInCluster(servers)
+	// Check the nodes are in the same VPC
+	networkID, subnetIDs, retErr := vngcloudutil.EnsureNodesInCluster(servers)
 	if retErr != nil {
-		plog.Errorf("All node are not in a same subnet: %v", retErr)
+		plog.Errorf("All node are not in a same VPC: %v", retErr)
 		return nil, retErr
 	}
-	networkID := vngcloudutil.GetNetworkID(servers, subnetID)
-	if networkID == "" {
-		klog.Errorf("Failed to get networkID from subnetID: %s", subnetID)
-		return nil, vErrors.ErrNetworkIDNotFound
-	}
-	subnet, err := vngcloudutil.GetSubnet(c.vServerSC, c.getProjectID(), networkID, subnetID)
+	subnet, err := vngcloudutil.GetSubnet(c.vServerSC, c.getProjectID(), networkID, subnetIDs[0])
 	if err != nil {
 		klog.Errorf("Failed to get subnet: %v", err)
 		return nil, err
 	}
+	ingressInspect.NetworkID = networkID
+	ingressInspect.SubnetID = subnetIDs[0]
 	ingressInspect.SubnetCIDR = subnet.CIDR
-	ingressInspect.LbOptions.SubnetID = subnetID
+	ingressInspect.LbOptions.SubnetID = subnetIDs[0]
 	ingressInspect.InstanceIDs = providerIDs
 
 	// Ensure pools and listener for this loadbalancer
@@ -633,6 +630,15 @@ func (c *vLB) ensureLoadBalancerInstance(inspect *Expander) (string, error) {
 		klog.Errorf("error when get lb: %v", err)
 		return inspect.serviceConf.LoadBalancerID, err
 	}
+	if inspect.SubnetID != lb.SubnetID {
+		subnet, err := vngcloudutil.GetSubnet(c.vServerSC, c.getProjectID(), inspect.NetworkID, lb.SubnetID)
+		if err != nil {
+			klog.Errorf("Failed to get subnet: %v", err)
+			return "", err
+		}
+		inspect.SubnetID = lb.SubnetID
+		inspect.SubnetCIDR = subnet.CIDR
+	}
 
 	checkDetailLB := func() {
 		if lb.Name != inspect.serviceConf.LoadBalancerName {