Merge pull request #4 from vngcloud/dev

Release 0.2.3

.github/workflows/ci-dev.yml

@@ -5,7 +5,7 @@ on:
     branches: ["dev"]
 
 env:
-  VERSION: v0.2.2
+  VERSION: v0.2.3
   REPO: vcr.vngcloud.vn/60108-annd2-ingress
 
 jobs:

.github/workflows/ci-main.yml

@@ -5,7 +5,7 @@ on:
     branches: ["main"]
 
 env:
-  VERSION: v0.2.2
+  VERSION: v0.2.3
   REPO: vcr.vngcloud.vn/81-vks-public
 
 jobs:

Makefile

@@ -42,7 +42,7 @@ TAR_FILE	?= rootfs.tar
 
 GOOS		?= $(shell go env GOOS)
 GOPROXY		?= $(shell go env GOPROXY)
-VERSION     ?= v0.2.2
+VERSION     ?= v0.2.3
 GOARCH		:=
 GOFLAGS		:=
 TAGS		:=

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.18.2
-	github.com/vngcloud/vngcloud-go-sdk v1.0.14-0.20240521072621-df4ad46f8a9b
+	github.com/vngcloud/vngcloud-go-sdk v1.0.14-0.20240620170114-c685b9ed20d0
 	gopkg.in/gcfg.v1 v1.2.3
 	k8s.io/api v0.29.0
 	k8s.io/apimachinery v0.29.0

go.sum

@@ -231,6 +231,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7
 github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk=
 github.com/vngcloud/vngcloud-go-sdk v1.0.14-0.20240521072621-df4ad46f8a9b h1:WUU4MuMeXakkmlD3Qlt0IanlNpxgTihTy1PrI9VWqDg=
 github.com/vngcloud/vngcloud-go-sdk v1.0.14-0.20240521072621-df4ad46f8a9b/go.mod h1:3ZjgN6oq5o7sYrShj2dOPOBF3cqWk6IW+/0VVpJWYf4=
+github.com/vngcloud/vngcloud-go-sdk v1.0.14-0.20240620170114-c685b9ed20d0 h1:OcWJ4GewAZsvfDExNp5RUYjJUBaA0KZ9neEx4rnGsEk=
+github.com/vngcloud/vngcloud-go-sdk v1.0.14-0.20240620170114-c685b9ed20d0/go.mod h1:3ZjgN6oq5o7sYrShj2dOPOBF3cqWk6IW+/0VVpJWYf4=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

pkg/ingress/controller/annotation.go

@@ -42,7 +42,7 @@ const (
 	ServiceAnnotationIdleTimeoutClient     = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/idle-timeout-client"     // both annotation and cloud-config
 	ServiceAnnotationIdleTimeoutMember     = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/idle-timeout-member"     // both annotation and cloud-config
 	ServiceAnnotationIdleTimeoutConnection = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/idle-timeout-connection" // both annotation and cloud-config
-	ServiceAnnotationInboundCIDRs          = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/inbound-cidrs"           // call only 1 time .......................
+	ServiceAnnotationInboundCIDRs          = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/inbound-cidrs"
 
 	// Pool annotations
 	ServiceAnnotationPoolAlgorithm       = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/pool-algorithm" // both annotation and cloud-config
@@ -78,7 +78,7 @@ type IngressConfig struct {
 	IdleTimeoutClient          int
 	IdleTimeoutMember          int
 	IdleTimeoutConnection      int
-	InboundCIDRs               string
+	InboundCIDRs               []string
 	HealthcheckProtocol        pool.CreateOptsHealthCheckProtocolOpt
 	HealthcheckHttpMethod      pool.CreateOptsHealthCheckMethodOpt
 	HealthcheckPath            string
@@ -110,7 +110,7 @@ func NewIngressConfig(pService *nwv1.Ingress) *IngressConfig {
 		IdleTimeoutClient:          50,
 		IdleTimeoutMember:          50,
 		IdleTimeoutConnection:      5,
-		InboundCIDRs:               "0.0.0.0/0",
+		InboundCIDRs:               []string{"0.0.0.0/0"},
 		HealthcheckProtocol:        pool.CreateOptsHealthCheckProtocolOptTCP,
 		HealthcheckHttpMethod:      pool.CreateOptsHealthCheckMethodOptGET,
 		HealthcheckPath:            "/",
@@ -162,7 +162,7 @@ func NewIngressConfig(pService *nwv1.Ingress) *IngressConfig {
 		opt.IdleTimeoutConnection = utils.ParseIntAnnotation(option, ServiceAnnotationIdleTimeoutConnection, opt.IdleTimeoutConnection)
 	}
 	if option, ok := pService.Annotations[ServiceAnnotationInboundCIDRs]; ok {
-		opt.InboundCIDRs = option
+		opt.InboundCIDRs = utils.ParseStringListAnnotation(option, ServiceAnnotationInboundCIDRs)
 	}
 
 	if option, ok := pService.Annotations[ServiceAnnotationHealthcheckProtocol]; ok {
@@ -313,7 +313,7 @@ func (s *IngressConfig) CreateListenerOptions(isHTTPS bool) *listener.CreateOpts
 		TimeoutClient:               s.IdleTimeoutClient,
 		TimeoutMember:               s.IdleTimeoutMember,
 		TimeoutConnection:           s.IdleTimeoutConnection,
-		AllowedCidrs:                s.InboundCIDRs,
+		AllowedCidrs:                utils.StringListToString(s.InboundCIDRs),
 	}
 	if isHTTPS {
 		opt.ListenerName = consts.DEFAULT_HTTPS_LISTENER_NAME

pkg/ingress/controller/controller.go

@@ -55,6 +55,7 @@ const (
 	CreateEvent EventType = "CREATE"
 	UpdateEvent EventType = "UPDATE"
 	DeleteEvent EventType = "DELETE"
+	SyncEvent   EventType = "SYNC"
 )
 
 // Event holds the context of an event
@@ -378,14 +379,9 @@ func (c *Controller) nodeSyncLoop() {
 		if !IsValid(&ing) {
 			continue
 		}
-
-		logrus.WithFields(logrus.Fields{"ingress": ing.Name, "namespace": ing.Namespace}).Debug("Starting to handle ingress")
-		err := c.ensureIngress(nil, &ing)
-		if err != nil {
-			logrus.WithFields(logrus.Fields{"ingress": ing.Name, "namespace": ing.Namespace}).Error("Failed to handle ingress:", err)
-			c.recorder.Event(&ing, apiv1.EventTypeWarning, "Failed", fmt.Sprintf("Failed to sync vngcloud resources for ingress %s: %v", fmt.Sprintf("%s/%s", ing.Namespace, ing.Name), err))
-			continue
-		}
+		copyIng := ing.DeepCopy()
+		logrus.WithFields(logrus.Fields{"ingress": copyIng.Name, "namespace": copyIng.Namespace}).Debug("Starting to sync ingress")
+		c.queue.AddRateLimited(Event{Obj: copyIng, Type: SyncEvent, oldObj: nil})
 	}
 	c.knownNodes = readyWorkerNodes
 	klog.Info("Finished to handle node change.")
@@ -452,7 +448,17 @@ func (c *Controller) processItem(event Event) {
 		} else {
 			c.recorder.Event(ing, apiv1.EventTypeNormal, "Deleted", fmt.Sprintf("Ingress %s", key))
 		}
+	case SyncEvent:
+		logger.Info("sync ingress")
+
+		if err := c.ensureIngress(oldIng, ing); err != nil {
+			utilruntime.HandleError(fmt.Errorf("failed to sync vngcloud resources for ingress %s: %v", key, err))
+			c.recorder.Event(ing, apiv1.EventTypeWarning, "Failed", fmt.Sprintf("Failed to sync vngcloud resources for ingress %s: %v", key, err))
+		} else {
+			c.recorder.Event(ing, apiv1.EventTypeNormal, "Synced", fmt.Sprintf("Ingress %s", key))
+		}
 	}
+
 	klog.Infoln("----- DONE -----")
 	// return nil
 }
@@ -684,6 +690,9 @@ func (c *Controller) DeleteLoadbalancer(ing *nwv1.Ingress) error {
 		}
 	}
 
+	// LB should active before delete
+	vngcloudutil.WaitForLBActive(c.vLBSC, c.getProjectID(), lbID)
+
 	canDeleteAllLB := func(lbID string) bool {
 		getPool, err := vngcloudutil.ListPoolOfLB(c.vLBSC, c.getProjectID(), lbID)
 		if err != nil {
@@ -826,7 +835,6 @@ func (c *Controller) inspectIngress(ing *nwv1.Ingress) (*Expander, error) {
 				PolicyExpander:       make([]*utils.PolicyExpander, 0),
 				PoolExpander:         make([]*utils.PoolExpander, 0),
 				ListenerExpander:     make([]*utils.ListenerExpander, 0),
-				SecurityGroups:       make([]string, 0),
 				SecGroupRuleExpander: make([]*utils.SecGroupRuleExpander, 0),
 			}}, nil
 	}
@@ -841,7 +849,6 @@ func (c *Controller) inspectIngress(ing *nwv1.Ingress) (*Expander, error) {
 		PolicyExpander:       make([]*utils.PolicyExpander, 0),
 		PoolExpander:         make([]*utils.PoolExpander, 0),
 		ListenerExpander:     make([]*utils.ListenerExpander, 0),
-		SecurityGroups:       make([]string, 0),
 		InstanceIDs:          make([]string, 0),
 		SecGroupRuleExpander: make([]*utils.SecGroupRuleExpander, 0),
 	}
@@ -880,21 +887,17 @@ func (c *Controller) inspectIngress(ing *nwv1.Ingress) (*Expander, error) {
 		klog.Errorf("All node are not in a same subnet: %v", retErr)
 		return nil, retErr
 	}
-	if option, ok := ing.Annotations[ServiceAnnotationInboundCIDRs]; ok {
-		ingressInspect.AllowCIDR = option
-	} else {
-		networkID := vngcloudutil.GetNetworkID(servers, subnetID)
-		if networkID == "" {
-			klog.Errorf("Failed to get networkID from subnetID: %s", subnetID)
-			return nil, vErrors.ErrNetworkIDNotFound
-		}
-		subnet, err := vngcloudutil.GetSubnet(c.vServerSC, c.getProjectID(), networkID, subnetID)
-		if err != nil {
-			klog.Errorf("Failed to get subnet: %v", err)
-			return nil, err
-		}
-		ingressInspect.AllowCIDR = subnet.CIDR
+	networkID := vngcloudutil.GetNetworkID(servers, subnetID)
+	if networkID == "" {
+		klog.Errorf("Failed to get networkID from subnetID: %s", subnetID)
+		return nil, vErrors.ErrNetworkIDNotFound
 	}
+	subnet, err := vngcloudutil.GetSubnet(c.vServerSC, c.getProjectID(), networkID, subnetID)
+	if err != nil {
+		klog.Errorf("Failed to get subnet: %v", err)
+		return nil, err
+	}
+	ingressInspect.SubnetCIDR = subnet.CIDR
 	ingressInspect.LbOptions.SubnetID = subnetID
 	ingressInspect.InstanceIDs = providerIDs
 
@@ -1086,7 +1089,7 @@ func (c *Controller) ensureLoadBalancerInstance(inspect *Expander) (string, erro
 		vngcloudutil.WaitForLBActive(c.vLBSC, c.getProjectID(), inspect.serviceConf.LoadBalancerID)
 	}
 
-	lb, err := vngcloudutil.GetLB(c.vLBSC, c.getProjectID(), inspect.serviceConf.LoadBalancerID)
+	lb, err := vngcloudutil.WaitForLBActive(c.vLBSC, c.getProjectID(), inspect.serviceConf.LoadBalancerID)
 	if err != nil {
 		klog.Errorf("error when get lb: %v", err)
 		return inspect.serviceConf.LoadBalancerID, err
@@ -1098,12 +1101,12 @@ func (c *Controller) ensureLoadBalancerInstance(inspect *Expander) (string, erro
 		}
 		if lb.PackageID != inspect.LbOptions.PackageID {
 			klog.Info("Resize load-balancer package to: ", inspect.LbOptions.PackageID)
-			err := vngcloudutil.ResizeLB(c.vLBSC, c.getProjectID(), inspect.serviceConf.LoadBalancerName, inspect.LbOptions.PackageID)
+			err := vngcloudutil.ResizeLB(c.vLBSC, c.getProjectID(), inspect.serviceConf.LoadBalancerID, inspect.LbOptions.PackageID)
 			if err != nil {
 				klog.Errorf("error when resize lb: %v", err)
 				return
 			}
-			vngcloudutil.WaitForLBActive(c.vLBSC, c.getProjectID(), inspect.serviceConf.LoadBalancerName)
+			vngcloudutil.WaitForLBActive(c.vLBSC, c.getProjectID(), inspect.serviceConf.LoadBalancerID)
 		}
 		if lb.Internal != (inspect.LbOptions.Scheme == loadbalancer.CreateOptsSchemeOptInternal) {
 			klog.Warning("Load balancer scheme not match, must delete and recreate")
@@ -1499,7 +1502,7 @@ func (c *Controller) ensureSecurityGroups(oldInspect, inspect *Expander) error {
 
 			for _, rule := range inspect.SecGroupRuleExpander {
 				rule.CreateOpts.SecurityGroupID = defaultSecgroup.UUID
-				rule.CreateOpts.RemoteIPPrefix = inspect.AllowCIDR
+				rule.CreateOpts.RemoteIPPrefix = inspect.SubnetCIDR
 			}
 
 			needDelete, needCreate := vngcloudutil.CompareSecgroupRule(secgroupRules, inspect.SecGroupRuleExpander)
@@ -1519,15 +1522,14 @@ func (c *Controller) ensureSecurityGroups(oldInspect, inspect *Expander) error {
 			}
 		}
 		ensureDefaultSecgroupRule()
-		inspect.SecurityGroups = []string{defaultSecgroup.UUID}
+		inspect.serviceConf.SecurityGroups = append(inspect.serviceConf.SecurityGroups, defaultSecgroup.UUID)
 	}
-	if len(inspect.SecurityGroups) < 1 || len(inspect.InstanceIDs) < 1 {
+	if len(inspect.serviceConf.SecurityGroups) < 1 || len(inspect.InstanceIDs) < 1 {
 		return nil
 	}
-
 	// add default security group to old inspect
 	if oldInspect != nil && oldInspect.serviceConf.IsAutoCreateSecurityGroup && defaultSecgroup != nil {
-		oldInspect.SecurityGroups = append(oldInspect.SecurityGroups, defaultSecgroup.UUID)
+		oldInspect.serviceConf.SecurityGroups = append(oldInspect.serviceConf.SecurityGroups, defaultSecgroup.UUID)
 	}
 
 	listSecgroups, err = vngcloudutil.ListSecurityGroups(c.vServerSC, c.getProjectID())
@@ -1542,7 +1544,7 @@ func (c *Controller) ensureSecurityGroups(oldInspect, inspect *Expander) error {
 	for _, secgroup := range listSecgroups {
 		mapSecgroups[secgroup.UUID] = true
 	}
-	for _, secgroup := range inspect.SecurityGroups {
+	for _, secgroup := range inspect.serviceConf.SecurityGroups {
 		if _, isHave := mapSecgroups[secgroup]; !isHave {
 			klog.Errorf("security group not found: %v", secgroup)
 		} else {
@@ -1571,7 +1573,7 @@ func (c *Controller) ensureSecurityGroups(oldInspect, inspect *Expander) error {
 		return err
 	}
 	for _, instanceID := range inspect.InstanceIDs {
-		err := ensureSecGroupsForInstance(instanceID, oldInspect.SecurityGroups, validSecgroups)
+		err := ensureSecGroupsForInstance(instanceID, oldInspect.serviceConf.SecurityGroups, validSecgroups)
 		if err != nil {
 			klog.Errorln("error when ensure security groups for instance", err)
 		}

pkg/utils/expander.go

@@ -53,12 +53,11 @@ type IngressInspect struct {
 	Name        string
 	Namespace   string
 	LbOptions   *loadbalancer.CreateOpts // create options for lb
-	AllowCIDR   string
+	SubnetCIDR  string
 
 	PolicyExpander       []*PolicyExpander
 	PoolExpander         []*PoolExpander
 	ListenerExpander     []*ListenerExpander
-	SecurityGroups       []string
 	InstanceIDs          []string
 	SecGroupRuleExpander []*SecGroupRuleExpander
 }

pkg/utils/utils.go

@@ -275,6 +275,10 @@ func ParseStringListAnnotation(s, annotation string) []string {
 	return validStr
 }
 
+func StringListToString(s []string) string {
+	return strings.Join(s, ",")
+}
+
 func ParseStringMapAnnotation(s, annotation string) map[string]string {
 	if s == "" {
 		return map[string]string{}
@@ -323,23 +327,23 @@ func getNodeConditionPredicate(node *apiv1.Node) bool {
 	}
 
 	// Deprecated in favor of LabelNodeExcludeLB, kept for consistency and will be removed later
-	if _, hasNodeRoleMasterLabel := node.Labels[consts.LabelNodeExcludeLB]; hasNodeRoleMasterLabel {
+	if _, hasNodeRoleMasterLabel := node.Labels[consts.DEFAULT_K8S_MASTER_LABEL]; hasNodeRoleMasterLabel {
 		return false
 	}
 
 	// If we have no info, don't accept
 	if len(node.Status.Conditions) == 0 {
 		return false
 	}
-	for _, cond := range node.Status.Conditions {
-		// We consider the node for load balancing only when its NodeReady condition status
-		// is ConditionTrue
-		if cond.Type == apiv1.NodeReady && cond.Status != apiv1.ConditionTrue {
-			klog.Info("ignoring node:", "name", node.Name, "status", cond.Status)
-			// log.WithFields(log.Fields{"name": node.Name, "status": cond.Status}).Info("ignoring node")
-			return false
-		}
-	}
+	// for _, cond := range node.Status.Conditions {
+	// 	// We consider the node for load balancing only when its NodeReady condition status
+	// 	// is ConditionTrue
+	// 	if cond.Type == apiv1.NodeReady && cond.Status != apiv1.ConditionTrue {
+	// 		klog.Info("ignoring node:", "name", node.Name, "status", cond.Status)
+	// 		// log.WithFields(log.Fields{"name": node.Name, "status": cond.Status}).Info("ignoring node")
+	// 		return false
+	// 	}
+	// }
 	return true
 }
 func ListServiceWithPredicate(serviceLister corelisters.ServiceLister) ([]*apiv1.Service, error) {

pkg/utils/vngcloud/loadbalancer.go

@@ -10,13 +10,13 @@ import (
 )
 
 func ListLBBySubnetID(client *client.ServiceClient, projectID string, subnetID string) ([]*objects.LoadBalancer, error) {
-	// klog.V(5).Infoln("[API] ListLBBySubnetID: ", "subnetID: ", subnetID)
+	klog.V(5).Infoln("[API] ListLBBySubnetID: ", "subnetID: ", subnetID)
 	opt := &loadbalancer.ListBySubnetIDOpts{}
 	opt.ProjectID = projectID
 	opt.SubnetID = subnetID
 
 	resp, err := loadbalancer.ListBySubnetID(client, opt)
-	// klog.V(5).Infoln("[API] ListLBBySubnetID: ", "resp: ", resp, "err: ", err)
+	klog.V(5).Infoln("[API] ListLBBySubnetID: ", "resp: ", resp, "err: ", err)
 	return resp, err
 }
 
@@ -31,21 +31,21 @@ func ListLB(client *client.ServiceClient, projectID string) ([]*objects.LoadBala
 }
 
 func GetLB(client *client.ServiceClient, projectID string, lbID string) (*objects.LoadBalancer, error) {
-	// klog.V(5).Infoln("[API] GetLB: ", "lbID: ", lbID)
+	klog.V(5).Infoln("[API] GetLB: ", "lbID: ", lbID)
 	opt := &loadbalancer.GetOpts{}
 	opt.ProjectID = projectID
 	opt.LoadBalancerID = lbID
 
 	resp, err := loadbalancer.Get(client, opt)
-	// klog.V(5).Infoln("[API] GetLB: ", "resp: ", resp, "err: ", err)
+	klog.V(5).Infoln("[API] GetLB: ", "resp: ", resp, "err: ", err)
 	if err != nil {
 		return resp, err.Error
 	}
 	return resp, nil
 }
 
 func CreateLB(client *client.ServiceClient, projectID string, lbOptions *loadbalancer.CreateOpts) (*objects.LoadBalancer, error) {
-	klog.V(5).Infoln("[API] CreateLB: ", "name: ", lbOptions.Name, "packageID: ", lbOptions.PackageID, "scheme: ", lbOptions.Scheme, "subnetID: ", lbOptions.SubnetID, "type: ", lbOptions.Type)
+	klog.V(5).Infoln("[API] CreateLB: ", "lbOptions: ", lbOptions)
 	lbOptions.ProjectID = projectID
 
 	resp, err := loadbalancer.Create(client, lbOptions)

pkg/utils/vngcloud/loadbalancer_certificate.go

@@ -16,11 +16,11 @@ func ImportCertificate(client *client.ServiceClient, projectID string, opt *cert
 }
 
 func ListCertificate(client *client.ServiceClient, projectID string) ([]*lObjects.Certificate, error) {
-	// klog.V(5).Infoln("[API] ListCertificate: ")
+	klog.V(5).Infoln("[API] ListCertificate: ")
 	opt := &certificates.ListOpts{}
 	opt.ProjectID = projectID
 	resp, err := certificates.List(client, opt)
-	// klog.V(5).Infoln("[API] ListCertificate: ", "resp: ", resp, "err: ", err)
+	klog.V(5).Infoln("[API] ListCertificate: ", "resp: ", resp, "err: ", err)
 	return resp, err
 }
 

pkg/utils/vngcloud/loadbalancer_listener.go

@@ -9,7 +9,7 @@ import (
 )
 
 func CreateListener(client *client.ServiceClient, projectID string, lbID string, opt *listener.CreateOpts) (*lObjects.Listener, error) {
-	klog.V(5).Infoln("[API] CreateListener: ", "lbID: ", lbID, opt.ListenerName, opt.DefaultPoolId, opt.ListenerProtocol, opt.AllowedCidrs, opt.ListenerProtocolPort, opt.TimeoutClient, opt.TimeoutConnection, opt.TimeoutMember, opt.CertificateAuthorities, opt.ClientCertificate, opt.DefaultCertificateAuthority)
+	klog.V(5).Infoln("[API] CreateListener: ", "lbID: ", lbID, "opt: ", opt)
 	opt.ProjectID = projectID
 	opt.LoadBalancerID = lbID
 
@@ -60,8 +60,7 @@ func DeleteListener(client *client.ServiceClient, projectID string, lbID, listen
 }
 
 func UpdateListener(client *client.ServiceClient, projectID string, lbID, listenerID string, opt *listener.UpdateOpts) error {
-	klog.V(5).Infoln("[API] UpdateListener: ", "lbID: ", lbID, "listenerID: ", listenerID)
-	klog.V(5).Infoln("[API] UpdateListener: ", "opt: ", opt)
+	klog.V(5).Infoln("[API] UpdateListener: ", "lbID: ", lbID, "listenerID: ", listenerID, "opt: ", opt)
 	opt.ProjectID = projectID
 	opt.LoadBalancerID = lbID
 	opt.ListenerID = listenerID