From ea327796ac1299898dbc0732d7c6a78ab1f587c9 Mon Sep 17 00:00:00 2001 From: andrew Date: Thu, 15 Aug 2024 18:30:03 +0700 Subject: [PATCH] =?UTF-8?q?=F0=9F=8C=B1=20add:=20support=20annotation=20en?= =?UTF-8?q?able-autoscale?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 2 +- go.mod | 7 +----- go.sum | 16 ++----------- pkg/ingress/controller/annotation.go | 35 +++++++++++----------------- pkg/ingress/controller/controller.go | 11 ++++++++- pkg/vngcloud/annotation.go | 17 ++++++++++---- pkg/vngcloud/vlb.go | 16 +++++++++++-- 7 files changed, 54 insertions(+), 50 deletions(-) diff --git a/Makefile b/Makefile index 68e1378..1710e14 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ GOFLAGS := TAGS := LDFLAGS := "-w -s -X 'k8s.io/component-base/version.gitVersion=$(VERSION)' -X 'github.com/vngcloud/cloud-provider-vngcloud/pkg/version.Version=$(VERSION)'" GOX_LDFLAGS := $(shell echo "$(LDFLAGS) -extldflags \"-static\"") -REGISTRY ?= vcr.vngcloud.vn/81-vks-public +REGISTRY ?= vcr.vngcloud.vn/60108-annd2-ingress IMAGE_OS ?= linux IMAGE_NAMES ?= vngcloud-controller-manager \ vngcloud-ingress-controller diff --git a/go.mod b/go.mod index cc0a22b..6f26c1c 100755 --- a/go.mod +++ b/go.mod @@ -11,8 +11,7 @@ require ( github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.18.2 - github.com/vngcloud/vngcloud-go-sdk v1.0.14-0.20240628025259-5f41d3d163b4 - github.com/wI2L/jsondiff v0.6.0 + github.com/vngcloud/vngcloud-go-sdk v1.0.14-0.20240815093737-8907a96f9716 gopkg.in/gcfg.v1 v1.2.3 k8s.io/api v0.30.1 k8s.io/apimachinery v0.30.1 @@ -92,10 +91,6 @@ require ( github.com/spf13/cast v1.6.0 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect - github.com/tidwall/gjson v1.17.1 // indirect - github.com/tidwall/match v1.1.1 // indirect - github.com/tidwall/pretty v1.2.1 // indirect - github.com/tidwall/sjson v1.2.5 // indirect go.etcd.io/etcd/api/v3 v3.5.10 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.10 // indirect go.etcd.io/etcd/client/v3 v3.5.10 // indirect diff --git a/go.sum b/go.sum index 8b1bdfb..a0b357c 100755 --- a/go.sum +++ b/go.sum @@ -220,22 +220,10 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= -github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= -github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= -github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= -github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= -github.com/vngcloud/vngcloud-go-sdk v1.0.14-0.20240628025259-5f41d3d163b4 h1:FdNMbxG4j8hpBQmSgn/Y8JEnWP4oml74VCVaLhStfJI= -github.com/vngcloud/vngcloud-go-sdk v1.0.14-0.20240628025259-5f41d3d163b4/go.mod h1:3ZjgN6oq5o7sYrShj2dOPOBF3cqWk6IW+/0VVpJWYf4= -github.com/wI2L/jsondiff v0.6.0 h1:zrsH3FbfVa3JO9llxrcDy/XLkYPLgoMX6Mz3T2PP2AI= -github.com/wI2L/jsondiff v0.6.0/go.mod h1:D6aQ5gKgPF9g17j+E9N7aasmU1O+XvfmWm1y8UMmNpw= +github.com/vngcloud/vngcloud-go-sdk v1.0.14-0.20240815093737-8907a96f9716 h1:ra119ks9SWEtIbwPUEr1+iyI5G6bd3vwIQx0yNaAO6U= +github.com/vngcloud/vngcloud-go-sdk v1.0.14-0.20240815093737-8907a96f9716/go.mod h1:3ZjgN6oq5o7sYrShj2dOPOBF3cqWk6IW+/0VVpJWYf4= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/pkg/ingress/controller/annotation.go b/pkg/ingress/controller/annotation.go index da6b879..e081a24 100644 --- a/pkg/ingress/controller/annotation.go +++ b/pkg/ingress/controller/annotation.go @@ -1,8 +1,6 @@ package controller import ( - "strconv" - "github.com/vngcloud/cloud-provider-vngcloud/pkg/consts" "github.com/vngcloud/cloud-provider-vngcloud/pkg/utils" "github.com/vngcloud/vngcloud-go-sdk/vngcloud/services/loadbalancer/v2/listener" @@ -37,6 +35,7 @@ const ( ServiceAnnotationTags = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/tags" ServiceAnnotationScheme = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/scheme" ServiceAnnotationCertificateIDs = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/certificate-ids" + ServiceAnnotationEnableAutoscale = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/enable-autoscale" // Listener annotations ServiceAnnotationIdleTimeoutClient = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/idle-timeout-client" // both annotation and cloud-config @@ -98,6 +97,7 @@ type IngressConfig struct { EnableStickySession bool EnableTLSEncryption bool CertificateIDs []string + EnableAutoscale bool } func NewIngressConfig(pService *nwv1.Ingress) *IngressConfig { @@ -130,6 +130,7 @@ func NewIngressConfig(pService *nwv1.Ingress) *IngressConfig { EnableStickySession: false, EnableTLSEncryption: false, CertificateIDs: []string{}, + EnableAutoscale: false, } if pService == nil { return opt @@ -257,22 +258,10 @@ func NewIngressConfig(pService *nwv1.Ingress) *IngressConfig { opt.HealthcheckPort = utils.ParseIntAnnotation(port, ServiceAnnotationHealthcheckPort, opt.HealthcheckPort) } if option, ok := pService.Annotations[ServiceAnnotationEnableStickySession]; ok { - switch option { - case "true", "false": - boolValue, _ := strconv.ParseBool(option) - opt.EnableStickySession = boolValue - default: - klog.Warningf("Invalid annotation \"%s\" value, must be true or false", ServiceAnnotationEnableStickySession) - } + opt.EnableStickySession = utils.ParseBoolAnnotation(option, ServiceAnnotationEnableStickySession, opt.EnableStickySession) } if option, ok := pService.Annotations[ServiceAnnotationEnableTLSEncryption]; ok { - switch option { - case "true", "false": - boolValue, _ := strconv.ParseBool(option) - opt.EnableTLSEncryption = boolValue - default: - klog.Warningf("Invalid annotation \"%s\" value, must be true or false", ServiceAnnotationEnableTLSEncryption) - } + opt.EnableTLSEncryption = utils.ParseBoolAnnotation(option, ServiceAnnotationEnableTLSEncryption, opt.EnableTLSEncryption) } if option, ok := pService.Annotations[ServiceAnnotationCertificateIDs]; ok { arr := utils.ParseStringListAnnotation(option, ServiceAnnotationCertificateIDs) @@ -287,16 +276,20 @@ func NewIngressConfig(pService *nwv1.Ingress) *IngressConfig { } opt.CertificateIDs = result } + if option, ok := pService.Annotations[ServiceAnnotationEnableAutoscale]; ok { + opt.EnableAutoscale = utils.ParseBoolAnnotation(option, ServiceAnnotationEnableAutoscale, opt.EnableAutoscale) + } return opt } func (s *IngressConfig) CreateLoadbalancerOptions() *loadbalancer.CreateOpts { opt := &loadbalancer.CreateOpts{ - Name: s.LoadBalancerName, - PackageID: s.PackageID, - Scheme: s.Scheme, - SubnetID: "", - Type: s.LoadBalancerType, + Name: s.LoadBalancerName, + PackageID: s.PackageID, + Scheme: s.Scheme, + SubnetID: "", + Type: s.LoadBalancerType, + AutoScalable: s.EnableAutoscale, } return opt } diff --git a/pkg/ingress/controller/controller.go b/pkg/ingress/controller/controller.go index db004b1..9fda631 100644 --- a/pkg/ingress/controller/controller.go +++ b/pkg/ingress/controller/controller.go @@ -3,6 +3,7 @@ package controller import ( "context" "fmt" + "net" "reflect" "strings" "sync" @@ -516,7 +517,12 @@ func (c *Controller) updateIngressStatus(ing *nwv1.Ingress, lb *lObjects.LoadBal newIng := latestIngress.DeepCopy() newState := new(nwv1.IngressLoadBalancerStatus) - newState.Ingress = []nwv1.IngressLoadBalancerIngress{{IP: lb.Address}} + addr := net.ParseIP(lb.Address) + if addr != nil { + newState.Ingress = []nwv1.IngressLoadBalancerIngress{{IP: lb.Address}} + } else { + newState.Ingress = []nwv1.IngressLoadBalancerIngress{{Hostname: lb.Address}} + } newIng.Status.LoadBalancer = *newState newObj, err := c.kubeClient.NetworkingV1().Ingresses(newIng.Namespace).UpdateStatus(context.TODO(), newIng, apimetav1.UpdateOptions{}) @@ -1127,6 +1133,9 @@ func (c *Controller) ensureLoadBalancerInstance(inspect *Expander) (string, erro if lb.Internal != (inspect.LbOptions.Scheme == loadbalancer.CreateOptsSchemeOptInternal) { klog.Warning("Load balancer scheme not match, must delete and recreate") } + if lb.AutoScalable != inspect.LbOptions.AutoScalable { + klog.Warning("Load balancer auto-scalable not match, must delete and recreate") + } } checkDetailLB() return inspect.serviceConf.LoadBalancerID, nil diff --git a/pkg/vngcloud/annotation.go b/pkg/vngcloud/annotation.go index 8c647c0..90948c3 100644 --- a/pkg/vngcloud/annotation.go +++ b/pkg/vngcloud/annotation.go @@ -35,6 +35,7 @@ const ( ServiceAnnotationSecurityGroups = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/security-groups" ServiceAnnotationTags = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/tags" ServiceAnnotationScheme = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/scheme" + ServiceAnnotationEnableAutoscale = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/enable-autoscale" // // Listener annotations ServiceAnnotationIdleTimeoutClient = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/idle-timeout-client" // both annotation and cloud-config @@ -95,6 +96,7 @@ type ServiceConfig struct { IsAutoCreateSecurityGroup bool SecurityGroups []string EnableProxyProtocol []string + EnableAutoscale bool } func NewServiceConfig(pService *apiv1.Service) *ServiceConfig { @@ -125,6 +127,7 @@ func NewServiceConfig(pService *apiv1.Service) *ServiceConfig { IsAutoCreateSecurityGroup: false, SecurityGroups: []string{}, EnableProxyProtocol: []string{}, + EnableAutoscale: false, } if pService == nil { return opt @@ -254,16 +257,20 @@ func NewServiceConfig(pService *apiv1.Service) *ServiceConfig { if proxy, ok := pService.Annotations[ServiceAnnotationProxyProtocol]; ok { opt.EnableProxyProtocol = utils.ParseStringListAnnotation(proxy, ServiceAnnotationProxyProtocol) } + if autoscale, ok := pService.Annotations[ServiceAnnotationEnableAutoscale]; ok { + opt.EnableAutoscale = utils.ParseBoolAnnotation(autoscale, ServiceAnnotationEnableAutoscale, opt.EnableAutoscale) + } return opt } func (s *ServiceConfig) CreateLoadbalancerOptions() *loadbalancer.CreateOpts { opt := &loadbalancer.CreateOpts{ - Name: s.LoadBalancerName, - PackageID: s.PackageID, - Scheme: s.Scheme, - SubnetID: "", - Type: s.LoadBalancerType, + Name: s.LoadBalancerName, + PackageID: s.PackageID, + Scheme: s.Scheme, + SubnetID: "", + Type: s.LoadBalancerType, + AutoScalable: s.EnableAutoscale, } return opt } diff --git a/pkg/vngcloud/vlb.go b/pkg/vngcloud/vlb.go index 0566937..a62283e 100644 --- a/pkg/vngcloud/vlb.go +++ b/pkg/vngcloud/vlb.go @@ -3,6 +3,7 @@ package vngcloud import ( "context" "fmt" + "net" "strings" "sync" "time" @@ -245,6 +246,10 @@ func (c *vLB) ensureLoadBalancer( } func (c *vLB) createLoadBalancerStatus(pService *lCoreV1.Service, lb *lObjects.LoadBalancer) *lCoreV1.LoadBalancerStatus { + if pService == nil { + klog.Warningln("can't createLoadBalancerStatus, service is nil") + return nil + } if pService.ObjectMeta.Annotations == nil { pService.ObjectMeta.Annotations = map[string]string{} } @@ -252,8 +257,12 @@ func (c *vLB) createLoadBalancerStatus(pService *lCoreV1.Service, lb *lObjects.L delete(pService.ObjectMeta.Annotations, ServiceAnnotationLoadBalancerName) status := &lCoreV1.LoadBalancerStatus{} - // Default to IP - status.Ingress = []lCoreV1.LoadBalancerIngress{{IP: lb.Address}} + addr := net.ParseIP(lb.Address) + if addr != nil { + status.Ingress = []lCoreV1.LoadBalancerIngress{{IP: lb.Address}} + } else { + status.Ingress = []lCoreV1.LoadBalancerIngress{{Hostname: lb.Address}} + } return status } @@ -641,6 +650,9 @@ func (c *vLB) ensureLoadBalancerInstance(inspect *Expander) (string, error) { if lb.Internal != (inspect.LbOptions.Scheme == loadbalancer.CreateOptsSchemeOptInternal) { klog.Warning("Load balancer scheme not match, must delete and recreate") } + if lb.AutoScalable != inspect.LbOptions.AutoScalable { + klog.Warning("Load balancer auto-scalable not match, must delete and recreate") + } } checkDetailLB() return inspect.serviceConf.LoadBalancerID, nil