Update SNI cert, tags, ignore annotation

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.18.2
-	github.com/vngcloud/vngcloud-go-sdk v1.0.6
+	github.com/vngcloud/vngcloud-go-sdk v1.0.14-0.20240521072621-df4ad46f8a9b
 	gopkg.in/gcfg.v1 v1.2.3
 	k8s.io/api v0.29.0
 	k8s.io/apimachinery v0.29.0

go.sum

@@ -231,6 +231,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7
 github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk=
 github.com/vngcloud/vngcloud-go-sdk v1.0.6 h1:AU7cNVUq0LZ2pNyKbr7+qfE+/+6U0GnbBvwGHRV+OYk=
 github.com/vngcloud/vngcloud-go-sdk v1.0.6/go.mod h1:3ZjgN6oq5o7sYrShj2dOPOBF3cqWk6IW+/0VVpJWYf4=
+github.com/vngcloud/vngcloud-go-sdk v1.0.14-0.20240521072621-df4ad46f8a9b h1:WUU4MuMeXakkmlD3Qlt0IanlNpxgTihTy1PrI9VWqDg=
+github.com/vngcloud/vngcloud-go-sdk v1.0.14-0.20240521072621-df4ad46f8a9b/go.mod h1:3ZjgN6oq5o7sYrShj2dOPOBF3cqWk6IW+/0VVpJWYf4=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

pkg/ingress/controller/annotation.go

@@ -18,6 +18,8 @@ const (
 
 // Annotations
 const (
+	ServiceAnnotationIgnore = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/ignore"
+
 	// ServiceAnnotationSubnetID              = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/subnet-id"  // both annotation and cloud-config
 	// ServiceAnnotationNetworkID             = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/network-id" // both annotation and cloud-config
 	// ServiceAnnotationOwnedListeners        = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/owned-listeners"
@@ -273,7 +275,17 @@ func NewIngressConfig(pService *nwv1.Ingress) *IngressConfig {
 		}
 	}
 	if option, ok := pService.Annotations[ServiceAnnotationCertificateIDs]; ok {
-		opt.CertificateIDs = utils.ParseStringListAnnotation(option, ServiceAnnotationCertificateIDs)
+		arr := utils.ParseStringListAnnotation(option, ServiceAnnotationCertificateIDs)
+		// remove duplicate certificate IDs
+		mapCertIDs := make(map[string]bool)
+		result := []string{}
+		for _, certID := range arr {
+			if !mapCertIDs[certID] {
+				result = append(result, certID)
+				mapCertIDs[certID] = true
+			}
+		}
+		opt.CertificateIDs = result
 	}
 	return opt
 }

pkg/ingress/controller/controller.go

@@ -458,6 +458,13 @@ func (c *Controller) processItem(event Event) {
 }
 
 func (c *Controller) deleteIngress(ing *nwv1.Ingress) error {
+	if option, ok := ing.Annotations[ServiceAnnotationIgnore]; ok {
+		if isIgnore := utils.ParseBoolAnnotation(option, ServiceAnnotationIgnore, false); isIgnore {
+			klog.Infof("Ignore ensure for service %s/%s", ing.Namespace, ing.Name)
+			return nil
+		}
+	}
+
 	err := c.DeleteLoadbalancer(ing)
 	if err != nil {
 		return err
@@ -466,6 +473,13 @@ func (c *Controller) deleteIngress(ing *nwv1.Ingress) error {
 }
 
 func (c *Controller) ensureIngress(oldIng, ing *nwv1.Ingress) error {
+	if option, ok := ing.Annotations[ServiceAnnotationIgnore]; ok {
+		if isIgnore := utils.ParseBoolAnnotation(option, ServiceAnnotationIgnore, false); isIgnore {
+			klog.Infof("Ignore ensure for service %s/%s", ing.Namespace, ing.Name)
+			return nil
+		}
+	}
+
 	lb, err := c.ensureCompareIngress(oldIng, ing)
 	if err != nil {
 		c.isReApplyNextTime = true
@@ -958,8 +972,12 @@ func (c *Controller) inspectIngress(ing *nwv1.Ingress) (*Expander, error) {
 			return nil, vErrors.ErrNoCertificateFound
 		} else {
 			listenerHttpsOpts := serviceConf.CreateListenerOptions(true)
-			listenerHttpsOpts.CertificateAuthorities = &(serviceConf.CertificateIDs)
 			listenerHttpsOpts.DefaultCertificateAuthority = &(serviceConf.CertificateIDs[0])
+			if len(serviceConf.CertificateIDs) > 1 {
+				listenerHttpsOpts.CertificateAuthorities = PointerOf[[]string](serviceConf.CertificateIDs[1:])
+			} else {
+				listenerHttpsOpts.CertificateAuthorities = PointerOf[[]string]([]string{})
+			}
 			listenerHttpsOpts.ClientCertificate = PointerOf[string]("")
 			ingressInspect.ListenerExpander = append(ingressInspect.ListenerExpander, &utils.ListenerExpander{
 				CreateOpts: *listenerHttpsOpts,
@@ -1060,6 +1078,10 @@ func (c *Controller) ensureLoadBalancerInstance(inspect *Expander) (string, erro
 			klog.Errorf("error when create new lb: %v", err)
 			return "", err
 		}
+		err = c.ensureTags(lb.UUID, inspect.serviceConf.Tags)
+		if err != nil {
+			klog.Errorln("error when ensure tags", err)
+		}
 		inspect.serviceConf.LoadBalancerID = lb.UUID
 		vngcloudutil.WaitForLBActive(c.vLBSC, c.getProjectID(), inspect.serviceConf.LoadBalancerID)
 	}
@@ -1095,6 +1117,11 @@ func (c *Controller) actionCompareIngress(lbID string, oldIngExpander, newIngExp
 	var err error
 	vngcloudutil.WaitForLBActive(c.vLBSC, c.getProjectID(), lbID)
 
+	err = c.ensureTags(lbID, newIngExpander.serviceConf.Tags)
+	if err != nil {
+		klog.Errorln("error when ensure tags", err)
+	}
+
 	curLBExpander, err := c.inspectCurrentLB(lbID)
 	if err != nil {
 		klog.Errorln("error when inspect current lb", err)
@@ -1192,10 +1219,6 @@ func (c *Controller) actionCompareIngress(lbID string, oldIngExpander, newIngExp
 	if err != nil {
 		klog.Errorln("error when ensure security groups", err)
 	}
-	err = c.ensureTags(lbID, newIngExpander.serviceConf.Tags)
-	if err != nil {
-		klog.Errorln("error when ensure security groups", err)
-	}
 
 	// delete redundant policy and pool if in oldIng
 	// with id from curLBExpander

pkg/utils/utils.go

@@ -251,6 +251,15 @@ func ParseIntAnnotation(s, annotation string, defaultValue int) int {
 	return i
 }
 
+func ParseBoolAnnotation(s, annotation string, defaultValue bool) bool {
+	b, err := strconv.ParseBool(s)
+	if err != nil {
+		klog.Warningf("Invalid annotation \"%s\" value, use default value = %t", annotation, defaultValue)
+		return defaultValue
+	}
+	return b
+}
+
 func ParseStringListAnnotation(s, annotation string) []string {
 	if s == "" {
 		return []string{}

pkg/utils/vngcloud/loadbalancer.go

@@ -61,8 +61,8 @@ func DeleteLB(client *client.ServiceClient, projectID string, lbID string) error
 
 	var err error
 	for {
-		err = loadbalancer.Delete(client, opt)
-		if err != nil && IsLoadBalancerNotReady(err) {
+		errSdk := loadbalancer.Delete(client, opt)
+		if errSdk != nil && IsLoadBalancerNotReady(errSdk.Error) {
 			klog.V(5).Infof("LoadBalancerNotReady, retry after 5 seconds")
 			time.Sleep(5 * time.Second)
 			continue

pkg/utils/vngcloud/loadbalancer_listener.go

@@ -46,8 +46,8 @@ func DeleteListener(client *client.ServiceClient, projectID string, lbID, listen
 
 	var err error
 	for {
-		err = listener.Delete(client, opt)
-		if err != nil && IsLoadBalancerNotReady(err) {
+		errSdk := listener.Delete(client, opt)
+		if errSdk != nil && IsLoadBalancerNotReady(errSdk.Error) {
 			klog.V(5).Infof("LoadBalancerNotReady, retry after 5 seconds")
 			time.Sleep(5 * time.Second)
 			continue

pkg/utils/vngcloud/loadbalancer_pool.go

@@ -101,8 +101,8 @@ func DeletePool(client *client.ServiceClient, projectID string, lbID, poolID str
 
 	var err error
 	for {
-		err = pool.Delete(client, opt)
-		if err != nil && IsLoadBalancerNotReady(err) {
+		errSdk := pool.Delete(client, opt)
+		if errSdk != nil && IsLoadBalancerNotReady(errSdk.Error) {
 			klog.V(5).Infof("LoadBalancerNotReady, retry after 5 seconds")
 			time.Sleep(5 * time.Second)
 			continue

pkg/utils/vngcloud/loadbalancer_utils.go

@@ -234,9 +234,7 @@ func CompareListenerOptions(ilis *lObjects.Listener, lisOptions *listener.Create
 		TimeoutConnection:           lisOptions.TimeoutConnection,
 		DefaultPoolId:               *lisOptions.DefaultPoolId,
 		DefaultCertificateAuthority: lisOptions.DefaultCertificateAuthority,
-		// Headers:                     lisOptions.Headers,
-		// ClientCertificate:           lisOptions.ClientCertificateAuthentication,
-		// ......................................... update later
+		CertificateAuthorities:      lisOptions.CertificateAuthorities,
 	}
 	if ilis.AllowedCidrs != lisOptions.AllowedCidrs ||
 		ilis.TimeoutClient != lisOptions.TimeoutClient ||
@@ -249,11 +247,33 @@ func CompareListenerOptions(ilis *lObjects.Listener, lisOptions *listener.Create
 		klog.Infof("listener need update default pool id: %s", *lisOptions.DefaultPoolId)
 		isNeedUpdate = true
 	}
-	if lisOptions.DefaultCertificateAuthority != nil && (ilis.DefaultCertificateAuthority == nil || *(ilis.DefaultCertificateAuthority) != *(lisOptions.DefaultCertificateAuthority)) {
+	if lisOptions.DefaultCertificateAuthority != nil &&
+		(ilis.DefaultCertificateAuthority == nil || *(ilis.DefaultCertificateAuthority) != *(lisOptions.DefaultCertificateAuthority)) {
 		klog.Infof("listener need update default certificate authority: %s", *lisOptions.DefaultCertificateAuthority)
 		isNeedUpdate = true
 	}
-	// update cert SNI here .......................................................
+
+	if len(ilis.CertificateAuthorities) > 0 && lisOptions.CertificateAuthorities == nil {
+		isNeedUpdate = true
+	} else if lisOptions.CertificateAuthorities != nil {
+		if len(ilis.CertificateAuthorities) != len(*lisOptions.CertificateAuthorities) {
+			klog.Infof("listener need update certificate authorities")
+			isNeedUpdate = true
+		} else {
+			maps := make(map[string]bool)
+			for _, ca := range ilis.CertificateAuthorities {
+				maps[ca] = true
+			}
+			for _, ca := range *lisOptions.CertificateAuthorities {
+				if _, ok := maps[ca]; !ok {
+					klog.Infof("listener need update certificate authorities")
+					isNeedUpdate = true
+					break
+				}
+			}
+		}
+	}
+
 	if !isNeedUpdate {
 		return nil
 	}

pkg/utils/vngcloud/security_group.go

@@ -58,5 +58,8 @@ func CreateSecurityGroup(client *client.ServiceClient, projectID string, name st
 	opt.Name = name
 	opt.Description = description
 	resp, err := secgroup.Create(client, opt)
-	return resp, err
+	if err != nil {
+		return nil, err.Error
+	}
+	return resp, nil
 }

pkg/vngcloud/annotation.go

@@ -17,6 +17,8 @@ const (
 
 // Annotations
 const (
+	ServiceAnnotationIgnore = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/ignore"
+
 	// ServiceAnnotationSubnetID              = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/subnet-id"  // both annotation and cloud-config
 	// ServiceAnnotationNetworkID             = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/network-id" // both annotation and cloud-config
 	// ServiceAnnotationOwnedListeners        = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/owned-listeners"

pkg/vngcloud/vlb.go

@@ -180,6 +180,13 @@ func (c *vLB) ensureLoadBalancer(
 	pCtx context.Context, clusterName string, pService *lCoreV1.Service, pNodes []*lCoreV1.Node) ( // params
 	rLb *lCoreV1.LoadBalancerStatus, rErr error) { // returns
 
+	if option, ok := pService.Annotations[ServiceAnnotationIgnore]; ok {
+		if isIgnore := utils.ParseBoolAnnotation(option, ServiceAnnotationIgnore, false); isIgnore {
+			klog.Infof("Ignore ensure for service %s/%s", pService.Namespace, pService.Name)
+			return nil, nil
+		}
+	}
+
 	// Patcher the service to prevent the service is updated by other controller
 	patcher := newServicePatcher(c.kubeClient, pService)
 	defer func() {
@@ -244,6 +251,13 @@ func (c *vLB) getProjectID() string {
 }
 
 func (c *vLB) ensureDeleteLoadBalancer(pCtx context.Context, clusterName string, pService *lCoreV1.Service) error {
+	if option, ok := pService.Annotations[ServiceAnnotationIgnore]; ok {
+		if isIgnore := utils.ParseBoolAnnotation(option, ServiceAnnotationIgnore, false); isIgnore {
+			klog.Infof("Ignore ensure for service %s/%s", pService.Namespace, pService.Name)
+			return nil
+		}
+	}
+
 	lbID, err := c.GetLoadbalancerIDByService(pService)
 	if lbID == "" {
 		klog.Infof("Not found lbID to delete")
@@ -587,6 +601,10 @@ func (c *vLB) ensureLoadBalancerInstance(inspect *Expander) (string, error) {
 			klog.Errorf("error when create new lb: %v", err)
 			return "", err
 		}
+		err = c.ensureTags(lb.UUID, inspect.serviceConf.Tags)
+		if err != nil {
+			klog.Errorln("error when ensure tags", err)
+		}
 		inspect.serviceConf.LoadBalancerID = lb.UUID
 		vngcloudutil.WaitForLBActive(c.vLBSC, c.getProjectID(), inspect.serviceConf.LoadBalancerID)
 	}
@@ -664,6 +682,11 @@ func (c *vLB) actionCompareIngress(lbID string, oldIngExpander, newIngExpander *
 	var err error
 	vngcloudutil.WaitForLBActive(c.vLBSC, c.getProjectID(), lbID)
 
+	err = c.ensureTags(lbID, newIngExpander.serviceConf.Tags)
+	if err != nil {
+		klog.Errorln("error when ensure tags", err)
+	}
+
 	// ensure all from newIngExpander
 	mapPoolNameIndex := make(map[string]int)
 	for poolIndex, ipool := range newIngExpander.PoolExpander {
@@ -702,10 +725,6 @@ func (c *vLB) actionCompareIngress(lbID string, oldIngExpander, newIngExpander *
 	if err != nil {
 		klog.Errorln("error when ensure security groups", err)
 	}
-	err = c.ensureTags(lbID, newIngExpander.serviceConf.Tags)
-	if err != nil {
-		klog.Errorln("error when ensure security groups", err)
-	}
 
 	// delete redundant policy and pool if in oldIng
 	// with id from curLBExpander