diff --git a/.github/workflows/ci-dev.yml b/.github/workflows/ci-dev.yml index 39202fe..f322c48 100644 --- a/.github/workflows/ci-dev.yml +++ b/.github/workflows/ci-dev.yml @@ -5,7 +5,7 @@ on: branches: ["dev"] env: - VERSION: v0.2.2 + VERSION: v0.2.3 REPO: vcr.vngcloud.vn/60108-annd2-ingress jobs: diff --git a/.github/workflows/ci-main.yml b/.github/workflows/ci-main.yml index b55df28..c628b6d 100644 --- a/.github/workflows/ci-main.yml +++ b/.github/workflows/ci-main.yml @@ -5,7 +5,7 @@ on: branches: ["main"] env: - VERSION: v0.2.2 + VERSION: v0.2.3 REPO: vcr.vngcloud.vn/81-vks-public jobs: diff --git a/Makefile b/Makefile index 3a744bf..80a5fb0 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ TAR_FILE ?= rootfs.tar GOOS ?= $(shell go env GOOS) GOPROXY ?= $(shell go env GOPROXY) -VERSION ?= v0.2.2 +VERSION ?= v0.2.3 GOARCH := GOFLAGS := TAGS := diff --git a/go.mod b/go.mod index 672bbec..88cff4e 100755 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.18.2 - github.com/vngcloud/vngcloud-go-sdk v1.0.14-0.20240521072621-df4ad46f8a9b + github.com/vngcloud/vngcloud-go-sdk v1.0.14-0.20240620170114-c685b9ed20d0 gopkg.in/gcfg.v1 v1.2.3 k8s.io/api v0.29.0 k8s.io/apimachinery v0.29.0 diff --git a/go.sum b/go.sum index 9583f2e..5c1d38e 100755 --- a/go.sum +++ b/go.sum @@ -231,6 +231,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7 github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/vngcloud/vngcloud-go-sdk v1.0.14-0.20240521072621-df4ad46f8a9b h1:WUU4MuMeXakkmlD3Qlt0IanlNpxgTihTy1PrI9VWqDg= github.com/vngcloud/vngcloud-go-sdk v1.0.14-0.20240521072621-df4ad46f8a9b/go.mod h1:3ZjgN6oq5o7sYrShj2dOPOBF3cqWk6IW+/0VVpJWYf4= +github.com/vngcloud/vngcloud-go-sdk v1.0.14-0.20240620170114-c685b9ed20d0 h1:OcWJ4GewAZsvfDExNp5RUYjJUBaA0KZ9neEx4rnGsEk= +github.com/vngcloud/vngcloud-go-sdk v1.0.14-0.20240620170114-c685b9ed20d0/go.mod h1:3ZjgN6oq5o7sYrShj2dOPOBF3cqWk6IW+/0VVpJWYf4= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/pkg/ingress/controller/annotation.go b/pkg/ingress/controller/annotation.go index 844a093..da6b879 100644 --- a/pkg/ingress/controller/annotation.go +++ b/pkg/ingress/controller/annotation.go @@ -42,7 +42,7 @@ const ( ServiceAnnotationIdleTimeoutClient = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/idle-timeout-client" // both annotation and cloud-config ServiceAnnotationIdleTimeoutMember = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/idle-timeout-member" // both annotation and cloud-config ServiceAnnotationIdleTimeoutConnection = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/idle-timeout-connection" // both annotation and cloud-config - ServiceAnnotationInboundCIDRs = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/inbound-cidrs" // call only 1 time ....................... + ServiceAnnotationInboundCIDRs = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/inbound-cidrs" // Pool annotations ServiceAnnotationPoolAlgorithm = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/pool-algorithm" // both annotation and cloud-config @@ -78,7 +78,7 @@ type IngressConfig struct { IdleTimeoutClient int IdleTimeoutMember int IdleTimeoutConnection int - InboundCIDRs string + InboundCIDRs []string HealthcheckProtocol pool.CreateOptsHealthCheckProtocolOpt HealthcheckHttpMethod pool.CreateOptsHealthCheckMethodOpt HealthcheckPath string @@ -110,7 +110,7 @@ func NewIngressConfig(pService *nwv1.Ingress) *IngressConfig { IdleTimeoutClient: 50, IdleTimeoutMember: 50, IdleTimeoutConnection: 5, - InboundCIDRs: "0.0.0.0/0", + InboundCIDRs: []string{"0.0.0.0/0"}, HealthcheckProtocol: pool.CreateOptsHealthCheckProtocolOptTCP, HealthcheckHttpMethod: pool.CreateOptsHealthCheckMethodOptGET, HealthcheckPath: "/", @@ -162,7 +162,7 @@ func NewIngressConfig(pService *nwv1.Ingress) *IngressConfig { opt.IdleTimeoutConnection = utils.ParseIntAnnotation(option, ServiceAnnotationIdleTimeoutConnection, opt.IdleTimeoutConnection) } if option, ok := pService.Annotations[ServiceAnnotationInboundCIDRs]; ok { - opt.InboundCIDRs = option + opt.InboundCIDRs = utils.ParseStringListAnnotation(option, ServiceAnnotationInboundCIDRs) } if option, ok := pService.Annotations[ServiceAnnotationHealthcheckProtocol]; ok { @@ -313,7 +313,7 @@ func (s *IngressConfig) CreateListenerOptions(isHTTPS bool) *listener.CreateOpts TimeoutClient: s.IdleTimeoutClient, TimeoutMember: s.IdleTimeoutMember, TimeoutConnection: s.IdleTimeoutConnection, - AllowedCidrs: s.InboundCIDRs, + AllowedCidrs: utils.StringListToString(s.InboundCIDRs), } if isHTTPS { opt.ListenerName = consts.DEFAULT_HTTPS_LISTENER_NAME diff --git a/pkg/ingress/controller/controller.go b/pkg/ingress/controller/controller.go index 092d4bc..41629e5 100644 --- a/pkg/ingress/controller/controller.go +++ b/pkg/ingress/controller/controller.go @@ -55,6 +55,7 @@ const ( CreateEvent EventType = "CREATE" UpdateEvent EventType = "UPDATE" DeleteEvent EventType = "DELETE" + SyncEvent EventType = "SYNC" ) // Event holds the context of an event @@ -378,14 +379,9 @@ func (c *Controller) nodeSyncLoop() { if !IsValid(&ing) { continue } - - logrus.WithFields(logrus.Fields{"ingress": ing.Name, "namespace": ing.Namespace}).Debug("Starting to handle ingress") - err := c.ensureIngress(nil, &ing) - if err != nil { - logrus.WithFields(logrus.Fields{"ingress": ing.Name, "namespace": ing.Namespace}).Error("Failed to handle ingress:", err) - c.recorder.Event(&ing, apiv1.EventTypeWarning, "Failed", fmt.Sprintf("Failed to sync vngcloud resources for ingress %s: %v", fmt.Sprintf("%s/%s", ing.Namespace, ing.Name), err)) - continue - } + copyIng := ing.DeepCopy() + logrus.WithFields(logrus.Fields{"ingress": copyIng.Name, "namespace": copyIng.Namespace}).Debug("Starting to sync ingress") + c.queue.AddRateLimited(Event{Obj: copyIng, Type: SyncEvent, oldObj: nil}) } c.knownNodes = readyWorkerNodes klog.Info("Finished to handle node change.") @@ -452,7 +448,17 @@ func (c *Controller) processItem(event Event) { } else { c.recorder.Event(ing, apiv1.EventTypeNormal, "Deleted", fmt.Sprintf("Ingress %s", key)) } + case SyncEvent: + logger.Info("sync ingress") + + if err := c.ensureIngress(oldIng, ing); err != nil { + utilruntime.HandleError(fmt.Errorf("failed to sync vngcloud resources for ingress %s: %v", key, err)) + c.recorder.Event(ing, apiv1.EventTypeWarning, "Failed", fmt.Sprintf("Failed to sync vngcloud resources for ingress %s: %v", key, err)) + } else { + c.recorder.Event(ing, apiv1.EventTypeNormal, "Synced", fmt.Sprintf("Ingress %s", key)) + } } + klog.Infoln("----- DONE -----") // return nil } @@ -684,6 +690,9 @@ func (c *Controller) DeleteLoadbalancer(ing *nwv1.Ingress) error { } } + // LB should active before delete + vngcloudutil.WaitForLBActive(c.vLBSC, c.getProjectID(), lbID) + canDeleteAllLB := func(lbID string) bool { getPool, err := vngcloudutil.ListPoolOfLB(c.vLBSC, c.getProjectID(), lbID) if err != nil { @@ -826,7 +835,6 @@ func (c *Controller) inspectIngress(ing *nwv1.Ingress) (*Expander, error) { PolicyExpander: make([]*utils.PolicyExpander, 0), PoolExpander: make([]*utils.PoolExpander, 0), ListenerExpander: make([]*utils.ListenerExpander, 0), - SecurityGroups: make([]string, 0), SecGroupRuleExpander: make([]*utils.SecGroupRuleExpander, 0), }}, nil } @@ -841,7 +849,6 @@ func (c *Controller) inspectIngress(ing *nwv1.Ingress) (*Expander, error) { PolicyExpander: make([]*utils.PolicyExpander, 0), PoolExpander: make([]*utils.PoolExpander, 0), ListenerExpander: make([]*utils.ListenerExpander, 0), - SecurityGroups: make([]string, 0), InstanceIDs: make([]string, 0), SecGroupRuleExpander: make([]*utils.SecGroupRuleExpander, 0), } @@ -880,21 +887,17 @@ func (c *Controller) inspectIngress(ing *nwv1.Ingress) (*Expander, error) { klog.Errorf("All node are not in a same subnet: %v", retErr) return nil, retErr } - if option, ok := ing.Annotations[ServiceAnnotationInboundCIDRs]; ok { - ingressInspect.AllowCIDR = option - } else { - networkID := vngcloudutil.GetNetworkID(servers, subnetID) - if networkID == "" { - klog.Errorf("Failed to get networkID from subnetID: %s", subnetID) - return nil, vErrors.ErrNetworkIDNotFound - } - subnet, err := vngcloudutil.GetSubnet(c.vServerSC, c.getProjectID(), networkID, subnetID) - if err != nil { - klog.Errorf("Failed to get subnet: %v", err) - return nil, err - } - ingressInspect.AllowCIDR = subnet.CIDR + networkID := vngcloudutil.GetNetworkID(servers, subnetID) + if networkID == "" { + klog.Errorf("Failed to get networkID from subnetID: %s", subnetID) + return nil, vErrors.ErrNetworkIDNotFound } + subnet, err := vngcloudutil.GetSubnet(c.vServerSC, c.getProjectID(), networkID, subnetID) + if err != nil { + klog.Errorf("Failed to get subnet: %v", err) + return nil, err + } + ingressInspect.SubnetCIDR = subnet.CIDR ingressInspect.LbOptions.SubnetID = subnetID ingressInspect.InstanceIDs = providerIDs @@ -1086,7 +1089,7 @@ func (c *Controller) ensureLoadBalancerInstance(inspect *Expander) (string, erro vngcloudutil.WaitForLBActive(c.vLBSC, c.getProjectID(), inspect.serviceConf.LoadBalancerID) } - lb, err := vngcloudutil.GetLB(c.vLBSC, c.getProjectID(), inspect.serviceConf.LoadBalancerID) + lb, err := vngcloudutil.WaitForLBActive(c.vLBSC, c.getProjectID(), inspect.serviceConf.LoadBalancerID) if err != nil { klog.Errorf("error when get lb: %v", err) return inspect.serviceConf.LoadBalancerID, err @@ -1098,12 +1101,12 @@ func (c *Controller) ensureLoadBalancerInstance(inspect *Expander) (string, erro } if lb.PackageID != inspect.LbOptions.PackageID { klog.Info("Resize load-balancer package to: ", inspect.LbOptions.PackageID) - err := vngcloudutil.ResizeLB(c.vLBSC, c.getProjectID(), inspect.serviceConf.LoadBalancerName, inspect.LbOptions.PackageID) + err := vngcloudutil.ResizeLB(c.vLBSC, c.getProjectID(), inspect.serviceConf.LoadBalancerID, inspect.LbOptions.PackageID) if err != nil { klog.Errorf("error when resize lb: %v", err) return } - vngcloudutil.WaitForLBActive(c.vLBSC, c.getProjectID(), inspect.serviceConf.LoadBalancerName) + vngcloudutil.WaitForLBActive(c.vLBSC, c.getProjectID(), inspect.serviceConf.LoadBalancerID) } if lb.Internal != (inspect.LbOptions.Scheme == loadbalancer.CreateOptsSchemeOptInternal) { klog.Warning("Load balancer scheme not match, must delete and recreate") @@ -1499,7 +1502,7 @@ func (c *Controller) ensureSecurityGroups(oldInspect, inspect *Expander) error { for _, rule := range inspect.SecGroupRuleExpander { rule.CreateOpts.SecurityGroupID = defaultSecgroup.UUID - rule.CreateOpts.RemoteIPPrefix = inspect.AllowCIDR + rule.CreateOpts.RemoteIPPrefix = inspect.SubnetCIDR } needDelete, needCreate := vngcloudutil.CompareSecgroupRule(secgroupRules, inspect.SecGroupRuleExpander) @@ -1519,15 +1522,14 @@ func (c *Controller) ensureSecurityGroups(oldInspect, inspect *Expander) error { } } ensureDefaultSecgroupRule() - inspect.SecurityGroups = []string{defaultSecgroup.UUID} + inspect.serviceConf.SecurityGroups = append(inspect.serviceConf.SecurityGroups, defaultSecgroup.UUID) } - if len(inspect.SecurityGroups) < 1 || len(inspect.InstanceIDs) < 1 { + if len(inspect.serviceConf.SecurityGroups) < 1 || len(inspect.InstanceIDs) < 1 { return nil } - // add default security group to old inspect if oldInspect != nil && oldInspect.serviceConf.IsAutoCreateSecurityGroup && defaultSecgroup != nil { - oldInspect.SecurityGroups = append(oldInspect.SecurityGroups, defaultSecgroup.UUID) + oldInspect.serviceConf.SecurityGroups = append(oldInspect.serviceConf.SecurityGroups, defaultSecgroup.UUID) } listSecgroups, err = vngcloudutil.ListSecurityGroups(c.vServerSC, c.getProjectID()) @@ -1542,7 +1544,7 @@ func (c *Controller) ensureSecurityGroups(oldInspect, inspect *Expander) error { for _, secgroup := range listSecgroups { mapSecgroups[secgroup.UUID] = true } - for _, secgroup := range inspect.SecurityGroups { + for _, secgroup := range inspect.serviceConf.SecurityGroups { if _, isHave := mapSecgroups[secgroup]; !isHave { klog.Errorf("security group not found: %v", secgroup) } else { @@ -1571,7 +1573,7 @@ func (c *Controller) ensureSecurityGroups(oldInspect, inspect *Expander) error { return err } for _, instanceID := range inspect.InstanceIDs { - err := ensureSecGroupsForInstance(instanceID, oldInspect.SecurityGroups, validSecgroups) + err := ensureSecGroupsForInstance(instanceID, oldInspect.serviceConf.SecurityGroups, validSecgroups) if err != nil { klog.Errorln("error when ensure security groups for instance", err) } diff --git a/pkg/utils/expander.go b/pkg/utils/expander.go index 819c54d..81e90a1 100644 --- a/pkg/utils/expander.go +++ b/pkg/utils/expander.go @@ -53,12 +53,11 @@ type IngressInspect struct { Name string Namespace string LbOptions *loadbalancer.CreateOpts // create options for lb - AllowCIDR string + SubnetCIDR string PolicyExpander []*PolicyExpander PoolExpander []*PoolExpander ListenerExpander []*ListenerExpander - SecurityGroups []string InstanceIDs []string SecGroupRuleExpander []*SecGroupRuleExpander } diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 5d157e6..19dbbd0 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -275,6 +275,10 @@ func ParseStringListAnnotation(s, annotation string) []string { return validStr } +func StringListToString(s []string) string { + return strings.Join(s, ",") +} + func ParseStringMapAnnotation(s, annotation string) map[string]string { if s == "" { return map[string]string{} @@ -323,7 +327,7 @@ func getNodeConditionPredicate(node *apiv1.Node) bool { } // Deprecated in favor of LabelNodeExcludeLB, kept for consistency and will be removed later - if _, hasNodeRoleMasterLabel := node.Labels[consts.LabelNodeExcludeLB]; hasNodeRoleMasterLabel { + if _, hasNodeRoleMasterLabel := node.Labels[consts.DEFAULT_K8S_MASTER_LABEL]; hasNodeRoleMasterLabel { return false } @@ -331,15 +335,15 @@ func getNodeConditionPredicate(node *apiv1.Node) bool { if len(node.Status.Conditions) == 0 { return false } - for _, cond := range node.Status.Conditions { - // We consider the node for load balancing only when its NodeReady condition status - // is ConditionTrue - if cond.Type == apiv1.NodeReady && cond.Status != apiv1.ConditionTrue { - klog.Info("ignoring node:", "name", node.Name, "status", cond.Status) - // log.WithFields(log.Fields{"name": node.Name, "status": cond.Status}).Info("ignoring node") - return false - } - } + // for _, cond := range node.Status.Conditions { + // // We consider the node for load balancing only when its NodeReady condition status + // // is ConditionTrue + // if cond.Type == apiv1.NodeReady && cond.Status != apiv1.ConditionTrue { + // klog.Info("ignoring node:", "name", node.Name, "status", cond.Status) + // // log.WithFields(log.Fields{"name": node.Name, "status": cond.Status}).Info("ignoring node") + // return false + // } + // } return true } func ListServiceWithPredicate(serviceLister corelisters.ServiceLister) ([]*apiv1.Service, error) { diff --git a/pkg/utils/vngcloud/loadbalancer.go b/pkg/utils/vngcloud/loadbalancer.go index f1065a0..ef17df7 100644 --- a/pkg/utils/vngcloud/loadbalancer.go +++ b/pkg/utils/vngcloud/loadbalancer.go @@ -10,13 +10,13 @@ import ( ) func ListLBBySubnetID(client *client.ServiceClient, projectID string, subnetID string) ([]*objects.LoadBalancer, error) { - // klog.V(5).Infoln("[API] ListLBBySubnetID: ", "subnetID: ", subnetID) + klog.V(5).Infoln("[API] ListLBBySubnetID: ", "subnetID: ", subnetID) opt := &loadbalancer.ListBySubnetIDOpts{} opt.ProjectID = projectID opt.SubnetID = subnetID resp, err := loadbalancer.ListBySubnetID(client, opt) - // klog.V(5).Infoln("[API] ListLBBySubnetID: ", "resp: ", resp, "err: ", err) + klog.V(5).Infoln("[API] ListLBBySubnetID: ", "resp: ", resp, "err: ", err) return resp, err } @@ -31,13 +31,13 @@ func ListLB(client *client.ServiceClient, projectID string) ([]*objects.LoadBala } func GetLB(client *client.ServiceClient, projectID string, lbID string) (*objects.LoadBalancer, error) { - // klog.V(5).Infoln("[API] GetLB: ", "lbID: ", lbID) + klog.V(5).Infoln("[API] GetLB: ", "lbID: ", lbID) opt := &loadbalancer.GetOpts{} opt.ProjectID = projectID opt.LoadBalancerID = lbID resp, err := loadbalancer.Get(client, opt) - // klog.V(5).Infoln("[API] GetLB: ", "resp: ", resp, "err: ", err) + klog.V(5).Infoln("[API] GetLB: ", "resp: ", resp, "err: ", err) if err != nil { return resp, err.Error } @@ -45,7 +45,7 @@ func GetLB(client *client.ServiceClient, projectID string, lbID string) (*object } func CreateLB(client *client.ServiceClient, projectID string, lbOptions *loadbalancer.CreateOpts) (*objects.LoadBalancer, error) { - klog.V(5).Infoln("[API] CreateLB: ", "name: ", lbOptions.Name, "packageID: ", lbOptions.PackageID, "scheme: ", lbOptions.Scheme, "subnetID: ", lbOptions.SubnetID, "type: ", lbOptions.Type) + klog.V(5).Infoln("[API] CreateLB: ", "lbOptions: ", lbOptions) lbOptions.ProjectID = projectID resp, err := loadbalancer.Create(client, lbOptions) diff --git a/pkg/utils/vngcloud/loadbalancer_certificate.go b/pkg/utils/vngcloud/loadbalancer_certificate.go index 2fe36a8..7ab2362 100644 --- a/pkg/utils/vngcloud/loadbalancer_certificate.go +++ b/pkg/utils/vngcloud/loadbalancer_certificate.go @@ -16,11 +16,11 @@ func ImportCertificate(client *client.ServiceClient, projectID string, opt *cert } func ListCertificate(client *client.ServiceClient, projectID string) ([]*lObjects.Certificate, error) { - // klog.V(5).Infoln("[API] ListCertificate: ") + klog.V(5).Infoln("[API] ListCertificate: ") opt := &certificates.ListOpts{} opt.ProjectID = projectID resp, err := certificates.List(client, opt) - // klog.V(5).Infoln("[API] ListCertificate: ", "resp: ", resp, "err: ", err) + klog.V(5).Infoln("[API] ListCertificate: ", "resp: ", resp, "err: ", err) return resp, err } diff --git a/pkg/utils/vngcloud/loadbalancer_listener.go b/pkg/utils/vngcloud/loadbalancer_listener.go index e273e15..8115134 100644 --- a/pkg/utils/vngcloud/loadbalancer_listener.go +++ b/pkg/utils/vngcloud/loadbalancer_listener.go @@ -9,7 +9,7 @@ import ( ) func CreateListener(client *client.ServiceClient, projectID string, lbID string, opt *listener.CreateOpts) (*lObjects.Listener, error) { - klog.V(5).Infoln("[API] CreateListener: ", "lbID: ", lbID, opt.ListenerName, opt.DefaultPoolId, opt.ListenerProtocol, opt.AllowedCidrs, opt.ListenerProtocolPort, opt.TimeoutClient, opt.TimeoutConnection, opt.TimeoutMember, opt.CertificateAuthorities, opt.ClientCertificate, opt.DefaultCertificateAuthority) + klog.V(5).Infoln("[API] CreateListener: ", "lbID: ", lbID, "opt: ", opt) opt.ProjectID = projectID opt.LoadBalancerID = lbID @@ -60,8 +60,7 @@ func DeleteListener(client *client.ServiceClient, projectID string, lbID, listen } func UpdateListener(client *client.ServiceClient, projectID string, lbID, listenerID string, opt *listener.UpdateOpts) error { - klog.V(5).Infoln("[API] UpdateListener: ", "lbID: ", lbID, "listenerID: ", listenerID) - klog.V(5).Infoln("[API] UpdateListener: ", "opt: ", opt) + klog.V(5).Infoln("[API] UpdateListener: ", "lbID: ", lbID, "listenerID: ", listenerID, "opt: ", opt) opt.ProjectID = projectID opt.LoadBalancerID = lbID opt.ListenerID = listenerID diff --git a/pkg/utils/vngcloud/loadbalancer_policy.go b/pkg/utils/vngcloud/loadbalancer_policy.go index 8cc7ac6..3a562a1 100644 --- a/pkg/utils/vngcloud/loadbalancer_policy.go +++ b/pkg/utils/vngcloud/loadbalancer_policy.go @@ -1,15 +1,16 @@ package vngcloud import ( + "time" + "github.com/vngcloud/vngcloud-go-sdk/client" lObjects "github.com/vngcloud/vngcloud-go-sdk/vngcloud/objects" "github.com/vngcloud/vngcloud-go-sdk/vngcloud/services/loadbalancer/v2/policy" "k8s.io/klog/v2" - "time" ) func CreatePolicy(client *client.ServiceClient, projectID string, lbID, listenerID string, opt *policy.CreateOptsBuilder) (*lObjects.Policy, error) { - klog.V(5).Infoln("[API] CreatePolicy: ", "lbID: ", lbID, "listenerID: ", listenerID) + klog.V(5).Infoln("[API] CreatePolicy: ", "lbID: ", lbID, "listenerID: ", listenerID, "opt: ", opt) opt.ProjectID = projectID opt.LoadBalancerID = lbID opt.ListenerID = listenerID @@ -26,35 +27,35 @@ func CreatePolicy(client *client.ServiceClient, projectID string, lbID, listener break } } - // klog.V(5).Infoln("[API] CreatePolicy: ", "resp: ", resp, "err: ", err) + klog.V(5).Infoln("[API] CreatePolicy: ", "resp: ", resp, "err: ", err) return resp, err } func ListPolicyOfListener(client *client.ServiceClient, projectID string, lbID, listenerID string) ([]*lObjects.Policy, error) { - // klog.V(5).Infoln("[API] ListPolicyOfListener: ", "lbID: ", lbID, "listenerID: ", listenerID) + klog.V(5).Infoln("[API] ListPolicyOfListener: ", "lbID: ", lbID, "listenerID: ", listenerID) opt := &policy.ListOptsBuilder{} opt.ProjectID = projectID opt.LoadBalancerID = lbID opt.ListenerID = listenerID resp, err := policy.List(client, opt) - // klog.V(5).Infoln("[API] ListPolicyOfListener: ", "resp: ", resp, "err: ", err) + klog.V(5).Infoln("[API] ListPolicyOfListener: ", "resp: ", resp, "err: ", err) return resp, err } func GetPolicy(client *client.ServiceClient, projectID string, lbID, listenerID, policyID string) (*lObjects.Policy, error) { - // klog.V(5).Infoln("[API] GetPolicy: ", "lbID: ", lbID, "listenerID: ", listenerID, "policyID: ", policyID) + klog.V(5).Infoln("[API] GetPolicy: ", "lbID: ", lbID, "listenerID: ", listenerID, "policyID: ", policyID) opt := &policy.GetOptsBuilder{} opt.ProjectID = projectID opt.LoadBalancerID = lbID opt.ListenerID = listenerID opt.PolicyID = policyID resp, err := policy.Get(client, opt) - // klog.V(5).Infoln("[API] GetPolicy: ", "resp: ", resp, "err: ", err) + klog.V(5).Infoln("[API] GetPolicy: ", "resp: ", resp, "err: ", err) return resp, err } func UpdatePolicy(client *client.ServiceClient, projectID string, lbID, listenerID, policyID string, opt *policy.UpdateOptsBuilder) error { - klog.V(5).Infoln("[API] UpdatePolicy: ", "lbID: ", lbID, "listenerID: ", listenerID, "policyID: ", policyID) + klog.V(5).Infoln("[API] UpdatePolicy: ", "lbID: ", lbID, "listenerID: ", listenerID, "policyID: ", policyID, "opt: ", opt) opt.ProjectID = projectID opt.LoadBalancerID = lbID opt.ListenerID = listenerID diff --git a/pkg/utils/vngcloud/loadbalancer_pool.go b/pkg/utils/vngcloud/loadbalancer_pool.go index f554528..f6178cb 100644 --- a/pkg/utils/vngcloud/loadbalancer_pool.go +++ b/pkg/utils/vngcloud/loadbalancer_pool.go @@ -1,15 +1,16 @@ package vngcloud import ( + "time" + "github.com/vngcloud/vngcloud-go-sdk/client" lObjects "github.com/vngcloud/vngcloud-go-sdk/vngcloud/objects" "github.com/vngcloud/vngcloud-go-sdk/vngcloud/services/loadbalancer/v2/pool" "k8s.io/klog/v2" - "time" ) func CreatePool(client *client.ServiceClient, projectID string, lbID string, opt *pool.CreateOpts) (*lObjects.Pool, error) { - klog.V(5).Infoln("[API] CreatePool: ", "lbID: ", lbID) + klog.V(5).Infoln("[API] CreatePool: ", "lbID: ", lbID, "opt: ", opt) opt.ProjectID = projectID opt.LoadBalancerID = lbID @@ -25,23 +26,23 @@ func CreatePool(client *client.ServiceClient, projectID string, lbID string, opt break } } - // klog.V(5).Infoln("[API] CreatePool: ", "resp: ", resp, "err: ", err) + klog.V(5).Infoln("[API] CreatePool: ", "resp: ", resp, "err: ", err) return resp, err } func ListPoolOfLB(client *client.ServiceClient, projectID string, lbID string) ([]*lObjects.Pool, error) { - // klog.V(5).Infoln("[API] ListPool: ", "lbID: ", lbID) + klog.V(5).Infoln("[API] ListPool: ", "lbID: ", lbID) opt := &pool.ListPoolsBasedLoadBalancerOpts{} opt.ProjectID = projectID opt.LoadBalancerID = lbID resp, err := pool.ListPoolsBasedLoadBalancer(client, opt) - // klog.V(5).Infoln("[API] ListPool: ", "resp: ", resp, "err: ", err) + klog.V(5).Infoln("[API] ListPool: ", "resp: ", resp, "err: ", err) return resp, err } func UpdatePoolMember(client *client.ServiceClient, projectID string, lbID, poolID string, mems []*pool.Member) error { - klog.V(5).Infoln("[API] UpdatePoolMember: ", "poolID: ", poolID) + klog.V(5).Infoln("[API] UpdatePoolMember: ", "poolID: ", poolID, "mems: ", mems) for _, mem := range mems { klog.V(5).Infof("[%s, %s, %d, %d]", mem.Name, mem.IpAddress, mem.Port, mem.MonitorPort) } @@ -64,31 +65,31 @@ func UpdatePoolMember(client *client.ServiceClient, projectID string, lbID, pool } } - // klog.V(5).Infoln("[API] UpdatePoolMember: ", "err: ", err) + klog.V(5).Infoln("[API] UpdatePoolMember: ", "err: ", err) return err } func GetPool(client *client.ServiceClient, projectID string, lbID, poolID string) (*lObjects.Pool, error) { - // klog.V(5).Infoln("[API] GetPool: ", "poolID: ", poolID) + klog.V(5).Infoln("[API] GetPool: ", "poolID: ", poolID) opt := &pool.GetOpts{} opt.ProjectID = projectID opt.LoadBalancerID = lbID opt.PoolID = poolID resp, err := pool.GetTotal(client, opt) - // klog.V(5).Infoln("[API] GetPool: ", "resp: ", resp, "err: ", err) + klog.V(5).Infoln("[API] GetPool: ", "resp: ", resp, "err: ", err) return resp, err } func GetMemberPool(client *client.ServiceClient, projectID string, lbID, poolID string) ([]*lObjects.Member, error) { - // klog.V(5).Infoln("[API] GetMemberPool: ", "poolID: ", poolID) + klog.V(5).Infoln("[API] GetMemberPool: ", "poolID: ", poolID) opt := &pool.GetMemberOpts{} opt.ProjectID = projectID opt.LoadBalancerID = lbID opt.PoolID = poolID resp, err := pool.GetMember(client, opt) - // klog.V(5).Infoln("[API] GetMemberPool: ", "resp: ", resp, "err: ", err) + klog.V(5).Infoln("[API] GetMemberPool: ", "resp: ", resp, "err: ", err) return resp, err } @@ -115,8 +116,7 @@ func DeletePool(client *client.ServiceClient, projectID string, lbID, poolID str } func UpdatePool(client *client.ServiceClient, projectID string, lbID, poolID string, opt *pool.UpdateOpts) error { - klog.V(5).Infoln("[API] UpdatePool: ", "lbID: ", lbID, "poolID: ", poolID) - klog.V(5).Infoln("[API] UpdatePool: ", "opt: ", opt) + klog.V(5).Infoln("[API] UpdatePool: ", "lbID: ", lbID, "poolID: ", poolID, "opt: ", opt) opt.ProjectID = projectID opt.LoadBalancerID = lbID opt.PoolID = poolID diff --git a/pkg/utils/vngcloud/loadbalancer_utils.go b/pkg/utils/vngcloud/loadbalancer_utils.go index 085b8da..82798a3 100644 --- a/pkg/utils/vngcloud/loadbalancer_utils.go +++ b/pkg/utils/vngcloud/loadbalancer_utils.go @@ -72,7 +72,7 @@ func FindListenerByPort(client *client.ServiceClient, projectID string, lbID str return nil, errors.ErrNotFound } -func WaitForLBActive(client *client.ServiceClient, projectID string, lbID string) { +func WaitForLBActive(client *client.ServiceClient, projectID string, lbID string) (*lObjects.LoadBalancer, error) { klog.Infof("Waiting for load balancer %s to be ready", lbID) var resultLb *lObjects.LoadBalancer @@ -109,7 +109,7 @@ func WaitForLBActive(client *client.ServiceClient, projectID string, lbID string klog.Errorf("timeout waiting for the loadbalancer %s with lb status %s", lbID, resultLb.Status) } - // return resultLb, err + return resultLb, err } func ComparePoolOptions(ipool *lObjects.Pool, poolOptions *pool.CreateOpts) *pool.UpdateOpts { @@ -235,6 +235,10 @@ func CompareListenerOptions(ilis *lObjects.Listener, lisOptions *listener.Create DefaultPoolId: *lisOptions.DefaultPoolId, DefaultCertificateAuthority: lisOptions.DefaultCertificateAuthority, CertificateAuthorities: lisOptions.CertificateAuthorities, + + // not support update these fields + Headers: ilis.Headers, // L7: if this field is nil, it will update empty ? => set it nil in L4 + ClientCertificate: nil, } if ilis.AllowedCidrs != lisOptions.AllowedCidrs || ilis.TimeoutClient != lisOptions.TimeoutClient || diff --git a/pkg/utils/vngcloud/security_group_rule.go b/pkg/utils/vngcloud/security_group_rule.go index 451f0b5..b28ea6f 100644 --- a/pkg/utils/vngcloud/security_group_rule.go +++ b/pkg/utils/vngcloud/security_group_rule.go @@ -11,7 +11,7 @@ import ( ) func CreateSecurityGroupRule(client *client.ServiceClient, projectID, secgroupID string, opts *secgroup_rule.CreateOpts) (*objects.SecgroupRule, error) { - klog.V(5).Infoln("[API] CreateSecurityGroupRule: ", "secgroupID: ", secgroupID) + klog.V(5).Infoln("[API] CreateSecurityGroupRule: ", "secgroupID: ", secgroupID, "opts: ", opts) opt := secgroup_rule.NewCreateOpts(projectID, secgroupID, opts) resp, err := secgroup_rule.Create(client, opt) return resp, err diff --git a/pkg/vngcloud/annotation.go b/pkg/vngcloud/annotation.go index aee90f4..8c647c0 100644 --- a/pkg/vngcloud/annotation.go +++ b/pkg/vngcloud/annotation.go @@ -40,7 +40,7 @@ const ( ServiceAnnotationIdleTimeoutClient = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/idle-timeout-client" // both annotation and cloud-config ServiceAnnotationIdleTimeoutMember = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/idle-timeout-member" // both annotation and cloud-config ServiceAnnotationIdleTimeoutConnection = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/idle-timeout-connection" // both annotation and cloud-config - ServiceAnnotationInboundCIDRs = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/inbound-cidrs" //.................... + ServiceAnnotationInboundCIDRs = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/inbound-cidrs" // // Pool annotations ServiceAnnotationPoolAlgorithm = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/pool-algorithm" // both annotation and cloud-config @@ -77,7 +77,7 @@ type ServiceConfig struct { IdleTimeoutClient int IdleTimeoutMember int IdleTimeoutConnection int - InboundCIDRs string + InboundCIDRs []string HealthcheckProtocol pool.CreateOptsHealthCheckProtocolOpt HealthcheckHttpMethod pool.CreateOptsHealthCheckMethodOpt HealthcheckPath string @@ -107,7 +107,7 @@ func NewServiceConfig(pService *apiv1.Service) *ServiceConfig { IdleTimeoutClient: 50, IdleTimeoutMember: 50, IdleTimeoutConnection: 5, - InboundCIDRs: "0.0.0.0/0", + InboundCIDRs: []string{"0.0.0.0/0"}, HealthcheckProtocol: pool.CreateOptsHealthCheckProtocolOptTCP, HealthcheckHttpMethod: pool.CreateOptsHealthCheckMethodOptGET, HealthcheckPath: "/", @@ -157,7 +157,7 @@ func NewServiceConfig(pService *apiv1.Service) *ServiceConfig { opt.IdleTimeoutConnection = utils.ParseIntAnnotation(option, ServiceAnnotationIdleTimeoutConnection, opt.IdleTimeoutConnection) } if option, ok := pService.Annotations[ServiceAnnotationInboundCIDRs]; ok { - opt.InboundCIDRs = option + opt.InboundCIDRs = utils.ParseStringListAnnotation(option, ServiceAnnotationInboundCIDRs) } if option, ok := pService.Annotations[ServiceAnnotationHealthcheckProtocol]; ok { @@ -280,7 +280,7 @@ func (s *ServiceConfig) CreateListenerOptions(pPort apiv1.ServicePort) *listener TimeoutClient: s.IdleTimeoutClient, TimeoutMember: s.IdleTimeoutMember, TimeoutConnection: s.IdleTimeoutConnection, - AllowedCidrs: s.InboundCIDRs, + AllowedCidrs: utils.StringListToString(s.InboundCIDRs), } return opt } diff --git a/pkg/vngcloud/vcontainer.go b/pkg/vngcloud/vcontainer.go index 75b628a..852c748 100644 --- a/pkg/vngcloud/vcontainer.go +++ b/pkg/vngcloud/vcontainer.go @@ -61,16 +61,17 @@ func (s *VContainer) LoadBalancer() (lcloudProvider.LoadBalancer, bool) { s.provider, "vserver-gateway") lb := &vLB{ - vLBSC: vlb, - vServerSC: vserver, - kubeClient: s.kubeClient, - eventRecorder: s.eventRecorder, - extraInfo: s.extraInfo, - vLbConfig: s.vLbOpts, - config: s.config, - trackLBUpdate: utils.NewUpdateTracker(), - serviceCache: make(map[string]*lCoreV1.Service), + vLBSC: vlb, + vServerSC: vserver, + kubeClient: s.kubeClient, + eventRecorder: s.eventRecorder, + extraInfo: s.extraInfo, + vLbConfig: s.vLbOpts, + config: s.config, + trackLBUpdate: utils.NewUpdateTracker(), + serviceCache: make(map[string]*lCoreV1.Service), isReApplyNextTime: false, + knownNodes: []*lCoreV1.Node{}, } go lb.Init() s.vlb = lb diff --git a/pkg/vngcloud/vlb.go b/pkg/vngcloud/vlb.go index e560b32..3af67b1 100644 --- a/pkg/vngcloud/vlb.go +++ b/pkg/vngcloud/vlb.go @@ -60,24 +60,26 @@ type ( vLB struct { vLBSC *lClient.ServiceClient vServerSC *lClient.ServiceClient + extraInfo *vngcloudutil.ExtraInfo kubeClient kubernetes.Interface eventRecorder record.EventRecorder vLbConfig VLbOpts - extraInfo *vngcloudutil.ExtraInfo - trackLBUpdate *utils.UpdateTracker serviceCache map[string]*lCoreV1.Service + knownNodes []*lCoreV1.Node serviceLister corelisters.ServiceLister serviceListerSynced cache.InformerSynced nodeLister corelisters.NodeLister nodeListerSynced cache.InformerSynced stopCh chan struct{} informer informers.SharedInformerFactory - mu sync.Mutex - numOfUpdatingThread int config *Config + isReApplyNextTime bool + trackLBUpdate *utils.UpdateTracker + mu sync.Mutex + numOfUpdatingThread int } // Config is the configuration for the VNG CLOUD load balancer controller, @@ -320,6 +322,9 @@ func (c *vLB) ensureDeleteLoadBalancer(pCtx context.Context, clusterName string, } } + // LB should active before delete + vngcloudutil.WaitForLBActive(c.vLBSC, c.getProjectID(), lbID) + canDeleteAllLB := func(lbID string) bool { getPool, err := vngcloudutil.ListPoolOfLB(c.vLBSC, c.getProjectID(), lbID) if err != nil { @@ -409,16 +414,20 @@ func (c *vLB) nodeSyncLoop() { } } - if !isReApply { - return - } - readyWorkerNodes, err := utils.ListNodeWithPredicate(c.nodeLister, make(map[string]string, 0)) if err != nil { klog.Errorf("Failed to retrieve current set of nodes from node lister: %v", err) c.isReApplyNextTime = true return } + if !isReApply && !utils.NodeSlicesEqual(readyWorkerNodes, c.knownNodes) { + isReApply = true + logrus.Infof("Detected change in list of current cluster nodes. Node set: %v", utils.NodeNames(readyWorkerNodes)) + } + + if !isReApply { + return + } services, err := utils.ListServiceWithPredicate(c.serviceLister) if err != nil { @@ -432,6 +441,7 @@ func (c *vLB) nodeSyncLoop() { klog.Errorf("Failed to reapply load balancer for service %s: %v", service.Name, err) } } + c.knownNodes = readyWorkerNodes } func (c *vLB) getClusterName() string { @@ -450,7 +460,6 @@ func (c *vLB) inspectService(pService *lCoreV1.Service) (*Expander, error) { PolicyExpander: make([]*utils.PolicyExpander, 0), PoolExpander: make([]*utils.PoolExpander, 0), ListenerExpander: make([]*utils.ListenerExpander, 0), - SecurityGroups: make([]string, 0), SecGroupRuleExpander: make([]*utils.SecGroupRuleExpander, 0), }, }, nil @@ -476,7 +485,6 @@ func (c *vLB) inspectService(pService *lCoreV1.Service) (*Expander, error) { PolicyExpander: make([]*utils.PolicyExpander, 0), PoolExpander: make([]*utils.PoolExpander, 0), ListenerExpander: make([]*utils.ListenerExpander, 0), - SecurityGroups: make([]string, 0), InstanceIDs: make([]string, 0), SecGroupRuleExpander: make([]*utils.SecGroupRuleExpander, 0), } @@ -515,21 +523,17 @@ func (c *vLB) inspectService(pService *lCoreV1.Service) (*Expander, error) { plog.Errorf("All node are not in a same subnet: %v", retErr) return nil, retErr } - if option, ok := pService.Annotations[ServiceAnnotationInboundCIDRs]; ok { - ingressInspect.AllowCIDR = option - } else { - networkID := vngcloudutil.GetNetworkID(servers, subnetID) - if networkID == "" { - klog.Errorf("Failed to get networkID from subnetID: %s", subnetID) - return nil, vErrors.ErrNetworkIDNotFound - } - subnet, err := vngcloudutil.GetSubnet(c.vServerSC, c.getProjectID(), networkID, subnetID) - if err != nil { - klog.Errorf("Failed to get subnet: %v", err) - return nil, err - } - ingressInspect.AllowCIDR = subnet.CIDR + networkID := vngcloudutil.GetNetworkID(servers, subnetID) + if networkID == "" { + klog.Errorf("Failed to get networkID from subnetID: %s", subnetID) + return nil, vErrors.ErrNetworkIDNotFound + } + subnet, err := vngcloudutil.GetSubnet(c.vServerSC, c.getProjectID(), networkID, subnetID) + if err != nil { + klog.Errorf("Failed to get subnet: %v", err) + return nil, err } + ingressInspect.SubnetCIDR = subnet.CIDR ingressInspect.LbOptions.SubnetID = subnetID ingressInspect.InstanceIDs = providerIDs @@ -610,7 +614,7 @@ func (c *vLB) ensureLoadBalancerInstance(inspect *Expander) (string, error) { vngcloudutil.WaitForLBActive(c.vLBSC, c.getProjectID(), inspect.serviceConf.LoadBalancerID) } - lb, err := vngcloudutil.GetLB(c.vLBSC, c.getProjectID(), inspect.serviceConf.LoadBalancerID) + lb, err := vngcloudutil.WaitForLBActive(c.vLBSC, c.getProjectID(), inspect.serviceConf.LoadBalancerID) if err != nil { klog.Errorf("error when get lb: %v", err) return inspect.serviceConf.LoadBalancerID, err @@ -871,6 +875,7 @@ func (c *vLB) ensureListenerV2(lbID, lisName string, listenerOpts listener.Creat updateOpts := vngcloudutil.CompareListenerOptions(lis, &listenerOpts) if updateOpts != nil { + updateOpts.Headers = nil err := vngcloudutil.UpdateListener(c.vLBSC, c.getProjectID(), lbID, lis.UUID, updateOpts) if err != nil { klog.Error("error when update listener: ", err) @@ -943,7 +948,7 @@ func (c *vLB) ensureSecurityGroups(oldInspect, inspect *Expander) error { for _, rule := range inspect.SecGroupRuleExpander { rule.CreateOpts.SecurityGroupID = defaultSecgroup.UUID - rule.CreateOpts.RemoteIPPrefix = inspect.AllowCIDR + rule.CreateOpts.RemoteIPPrefix = inspect.SubnetCIDR } needDelete, needCreate := vngcloudutil.CompareSecgroupRule(secgroupRules, inspect.SecGroupRuleExpander) @@ -963,15 +968,15 @@ func (c *vLB) ensureSecurityGroups(oldInspect, inspect *Expander) error { } } ensureDefaultSecgroupRule() - inspect.SecurityGroups = []string{defaultSecgroup.UUID} + inspect.serviceConf.SecurityGroups = append(inspect.serviceConf.SecurityGroups, defaultSecgroup.UUID) } - if len(inspect.SecurityGroups) < 1 || len(inspect.InstanceIDs) < 1 { + if len(inspect.serviceConf.SecurityGroups) < 1 || len(inspect.InstanceIDs) < 1 { return nil } // add default security group to old inspect if oldInspect != nil && oldInspect.serviceConf.IsAutoCreateSecurityGroup && defaultSecgroup != nil { - oldInspect.SecurityGroups = append(oldInspect.SecurityGroups, defaultSecgroup.UUID) + oldInspect.serviceConf.SecurityGroups = append(oldInspect.serviceConf.SecurityGroups, defaultSecgroup.UUID) } listSecgroups, err = vngcloudutil.ListSecurityGroups(c.vServerSC, c.getProjectID()) @@ -986,7 +991,7 @@ func (c *vLB) ensureSecurityGroups(oldInspect, inspect *Expander) error { for _, secgroup := range listSecgroups { mapSecgroups[secgroup.UUID] = true } - for _, secgroup := range inspect.SecurityGroups { + for _, secgroup := range inspect.serviceConf.SecurityGroups { if _, isHave := mapSecgroups[secgroup]; !isHave { klog.Errorf("security group not found: %v", secgroup) } else { @@ -1015,7 +1020,7 @@ func (c *vLB) ensureSecurityGroups(oldInspect, inspect *Expander) error { return err } for _, instanceID := range inspect.InstanceIDs { - err := ensureSecGroupsForInstance(instanceID, oldInspect.SecurityGroups, validSecgroups) + err := ensureSecGroupsForInstance(instanceID, oldInspect.serviceConf.SecurityGroups, validSecgroups) if err != nil { klog.Errorln("error when ensure security groups for instance", err) }