Skip to content

Commit

Permalink
🌱 add: support annotation enable-autoscale
Browse files Browse the repository at this point in the history
  • Loading branch information
anngdinh committed Aug 15, 2024
1 parent 09bfecb commit ea32779
Show file tree
Hide file tree
Showing 7 changed files with 54 additions and 50 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ GOFLAGS :=
TAGS :=
LDFLAGS := "-w -s -X 'k8s.io/component-base/version.gitVersion=$(VERSION)' -X 'github.com/vngcloud/cloud-provider-vngcloud/pkg/version.Version=$(VERSION)'"
GOX_LDFLAGS := $(shell echo "$(LDFLAGS) -extldflags \"-static\"")
REGISTRY ?= vcr.vngcloud.vn/81-vks-public
REGISTRY ?= vcr.vngcloud.vn/60108-annd2-ingress
IMAGE_OS ?= linux
IMAGE_NAMES ?= vngcloud-controller-manager \
vngcloud-ingress-controller
Expand Down
7 changes: 1 addition & 6 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,7 @@ require (
github.com/spf13/cobra v1.7.0
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.18.2
github.com/vngcloud/vngcloud-go-sdk v1.0.14-0.20240628025259-5f41d3d163b4
github.com/wI2L/jsondiff v0.6.0
github.com/vngcloud/vngcloud-go-sdk v1.0.14-0.20240815093737-8907a96f9716
gopkg.in/gcfg.v1 v1.2.3
k8s.io/api v0.30.1
k8s.io/apimachinery v0.30.1
Expand Down Expand Up @@ -92,10 +91,6 @@ require (
github.com/spf13/cast v1.6.0 // indirect
github.com/stoewer/go-strcase v1.2.0 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/tidwall/gjson v1.17.1 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
github.com/tidwall/sjson v1.2.5 // indirect
go.etcd.io/etcd/api/v3 v3.5.10 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.10 // indirect
go.etcd.io/etcd/client/v3 v3.5.10 // indirect
Expand Down
16 changes: 2 additions & 14 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -220,22 +220,10 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U=
github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE=
github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk=
github.com/vngcloud/vngcloud-go-sdk v1.0.14-0.20240628025259-5f41d3d163b4 h1:FdNMbxG4j8hpBQmSgn/Y8JEnWP4oml74VCVaLhStfJI=
github.com/vngcloud/vngcloud-go-sdk v1.0.14-0.20240628025259-5f41d3d163b4/go.mod h1:3ZjgN6oq5o7sYrShj2dOPOBF3cqWk6IW+/0VVpJWYf4=
github.com/wI2L/jsondiff v0.6.0 h1:zrsH3FbfVa3JO9llxrcDy/XLkYPLgoMX6Mz3T2PP2AI=
github.com/wI2L/jsondiff v0.6.0/go.mod h1:D6aQ5gKgPF9g17j+E9N7aasmU1O+XvfmWm1y8UMmNpw=
github.com/vngcloud/vngcloud-go-sdk v1.0.14-0.20240815093737-8907a96f9716 h1:ra119ks9SWEtIbwPUEr1+iyI5G6bd3vwIQx0yNaAO6U=
github.com/vngcloud/vngcloud-go-sdk v1.0.14-0.20240815093737-8907a96f9716/go.mod h1:3ZjgN6oq5o7sYrShj2dOPOBF3cqWk6IW+/0VVpJWYf4=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
Expand Down
35 changes: 14 additions & 21 deletions pkg/ingress/controller/annotation.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package controller

import (
"strconv"

"github.com/vngcloud/cloud-provider-vngcloud/pkg/consts"
"github.com/vngcloud/cloud-provider-vngcloud/pkg/utils"
"github.com/vngcloud/vngcloud-go-sdk/vngcloud/services/loadbalancer/v2/listener"
Expand Down Expand Up @@ -37,6 +35,7 @@ const (
ServiceAnnotationTags = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/tags"
ServiceAnnotationScheme = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/scheme"
ServiceAnnotationCertificateIDs = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/certificate-ids"
ServiceAnnotationEnableAutoscale = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/enable-autoscale"

// Listener annotations
ServiceAnnotationIdleTimeoutClient = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/idle-timeout-client" // both annotation and cloud-config
Expand Down Expand Up @@ -98,6 +97,7 @@ type IngressConfig struct {
EnableStickySession bool
EnableTLSEncryption bool
CertificateIDs []string
EnableAutoscale bool
}

func NewIngressConfig(pService *nwv1.Ingress) *IngressConfig {
Expand Down Expand Up @@ -130,6 +130,7 @@ func NewIngressConfig(pService *nwv1.Ingress) *IngressConfig {
EnableStickySession: false,
EnableTLSEncryption: false,
CertificateIDs: []string{},
EnableAutoscale: false,
}
if pService == nil {
return opt
Expand Down Expand Up @@ -257,22 +258,10 @@ func NewIngressConfig(pService *nwv1.Ingress) *IngressConfig {
opt.HealthcheckPort = utils.ParseIntAnnotation(port, ServiceAnnotationHealthcheckPort, opt.HealthcheckPort)
}
if option, ok := pService.Annotations[ServiceAnnotationEnableStickySession]; ok {
switch option {
case "true", "false":
boolValue, _ := strconv.ParseBool(option)
opt.EnableStickySession = boolValue
default:
klog.Warningf("Invalid annotation \"%s\" value, must be true or false", ServiceAnnotationEnableStickySession)
}
opt.EnableStickySession = utils.ParseBoolAnnotation(option, ServiceAnnotationEnableStickySession, opt.EnableStickySession)
}
if option, ok := pService.Annotations[ServiceAnnotationEnableTLSEncryption]; ok {
switch option {
case "true", "false":
boolValue, _ := strconv.ParseBool(option)
opt.EnableTLSEncryption = boolValue
default:
klog.Warningf("Invalid annotation \"%s\" value, must be true or false", ServiceAnnotationEnableTLSEncryption)
}
opt.EnableTLSEncryption = utils.ParseBoolAnnotation(option, ServiceAnnotationEnableTLSEncryption, opt.EnableTLSEncryption)
}
if option, ok := pService.Annotations[ServiceAnnotationCertificateIDs]; ok {
arr := utils.ParseStringListAnnotation(option, ServiceAnnotationCertificateIDs)
Expand All @@ -287,16 +276,20 @@ func NewIngressConfig(pService *nwv1.Ingress) *IngressConfig {
}
opt.CertificateIDs = result
}
if option, ok := pService.Annotations[ServiceAnnotationEnableAutoscale]; ok {
opt.EnableAutoscale = utils.ParseBoolAnnotation(option, ServiceAnnotationEnableAutoscale, opt.EnableAutoscale)
}
return opt
}

func (s *IngressConfig) CreateLoadbalancerOptions() *loadbalancer.CreateOpts {
opt := &loadbalancer.CreateOpts{
Name: s.LoadBalancerName,
PackageID: s.PackageID,
Scheme: s.Scheme,
SubnetID: "",
Type: s.LoadBalancerType,
Name: s.LoadBalancerName,
PackageID: s.PackageID,
Scheme: s.Scheme,
SubnetID: "",
Type: s.LoadBalancerType,
AutoScalable: s.EnableAutoscale,
}
return opt
}
Expand Down
11 changes: 10 additions & 1 deletion pkg/ingress/controller/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package controller
import (
"context"
"fmt"
"net"
"reflect"
"strings"
"sync"
Expand Down Expand Up @@ -516,7 +517,12 @@ func (c *Controller) updateIngressStatus(ing *nwv1.Ingress, lb *lObjects.LoadBal

newIng := latestIngress.DeepCopy()
newState := new(nwv1.IngressLoadBalancerStatus)
newState.Ingress = []nwv1.IngressLoadBalancerIngress{{IP: lb.Address}}
addr := net.ParseIP(lb.Address)
if addr != nil {
newState.Ingress = []nwv1.IngressLoadBalancerIngress{{IP: lb.Address}}
} else {
newState.Ingress = []nwv1.IngressLoadBalancerIngress{{Hostname: lb.Address}}
}
newIng.Status.LoadBalancer = *newState

newObj, err := c.kubeClient.NetworkingV1().Ingresses(newIng.Namespace).UpdateStatus(context.TODO(), newIng, apimetav1.UpdateOptions{})
Expand Down Expand Up @@ -1127,6 +1133,9 @@ func (c *Controller) ensureLoadBalancerInstance(inspect *Expander) (string, erro
if lb.Internal != (inspect.LbOptions.Scheme == loadbalancer.CreateOptsSchemeOptInternal) {
klog.Warning("Load balancer scheme not match, must delete and recreate")
}
if lb.AutoScalable != inspect.LbOptions.AutoScalable {
klog.Warning("Load balancer auto-scalable not match, must delete and recreate")
}
}
checkDetailLB()
return inspect.serviceConf.LoadBalancerID, nil
Expand Down
17 changes: 12 additions & 5 deletions pkg/vngcloud/annotation.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ const (
ServiceAnnotationSecurityGroups = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/security-groups"
ServiceAnnotationTags = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/tags"
ServiceAnnotationScheme = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/scheme"
ServiceAnnotationEnableAutoscale = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/enable-autoscale"

// // Listener annotations
ServiceAnnotationIdleTimeoutClient = DEFAULT_K8S_SERVICE_ANNOTATION_PREFIX + "/idle-timeout-client" // both annotation and cloud-config
Expand Down Expand Up @@ -95,6 +96,7 @@ type ServiceConfig struct {
IsAutoCreateSecurityGroup bool
SecurityGroups []string
EnableProxyProtocol []string
EnableAutoscale bool
}

func NewServiceConfig(pService *apiv1.Service) *ServiceConfig {
Expand Down Expand Up @@ -125,6 +127,7 @@ func NewServiceConfig(pService *apiv1.Service) *ServiceConfig {
IsAutoCreateSecurityGroup: false,
SecurityGroups: []string{},
EnableProxyProtocol: []string{},
EnableAutoscale: false,
}
if pService == nil {
return opt
Expand Down Expand Up @@ -254,16 +257,20 @@ func NewServiceConfig(pService *apiv1.Service) *ServiceConfig {
if proxy, ok := pService.Annotations[ServiceAnnotationProxyProtocol]; ok {
opt.EnableProxyProtocol = utils.ParseStringListAnnotation(proxy, ServiceAnnotationProxyProtocol)
}
if autoscale, ok := pService.Annotations[ServiceAnnotationEnableAutoscale]; ok {
opt.EnableAutoscale = utils.ParseBoolAnnotation(autoscale, ServiceAnnotationEnableAutoscale, opt.EnableAutoscale)
}
return opt
}

func (s *ServiceConfig) CreateLoadbalancerOptions() *loadbalancer.CreateOpts {
opt := &loadbalancer.CreateOpts{
Name: s.LoadBalancerName,
PackageID: s.PackageID,
Scheme: s.Scheme,
SubnetID: "",
Type: s.LoadBalancerType,
Name: s.LoadBalancerName,
PackageID: s.PackageID,
Scheme: s.Scheme,
SubnetID: "",
Type: s.LoadBalancerType,
AutoScalable: s.EnableAutoscale,
}
return opt
}
Expand Down
16 changes: 14 additions & 2 deletions pkg/vngcloud/vlb.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package vngcloud
import (
"context"
"fmt"
"net"
"strings"
"sync"
"time"
Expand Down Expand Up @@ -245,15 +246,23 @@ func (c *vLB) ensureLoadBalancer(
}

func (c *vLB) createLoadBalancerStatus(pService *lCoreV1.Service, lb *lObjects.LoadBalancer) *lCoreV1.LoadBalancerStatus {
if pService == nil {
klog.Warningln("can't createLoadBalancerStatus, service is nil")
return nil
}
if pService.ObjectMeta.Annotations == nil {
pService.ObjectMeta.Annotations = map[string]string{}
}
pService.ObjectMeta.Annotations[ServiceAnnotationLoadBalancerID] = lb.UUID
delete(pService.ObjectMeta.Annotations, ServiceAnnotationLoadBalancerName)

status := &lCoreV1.LoadBalancerStatus{}
// Default to IP
status.Ingress = []lCoreV1.LoadBalancerIngress{{IP: lb.Address}}
addr := net.ParseIP(lb.Address)
if addr != nil {
status.Ingress = []lCoreV1.LoadBalancerIngress{{IP: lb.Address}}
} else {
status.Ingress = []lCoreV1.LoadBalancerIngress{{Hostname: lb.Address}}
}
return status
}

Expand Down Expand Up @@ -641,6 +650,9 @@ func (c *vLB) ensureLoadBalancerInstance(inspect *Expander) (string, error) {
if lb.Internal != (inspect.LbOptions.Scheme == loadbalancer.CreateOptsSchemeOptInternal) {
klog.Warning("Load balancer scheme not match, must delete and recreate")
}
if lb.AutoScalable != inspect.LbOptions.AutoScalable {
klog.Warning("Load balancer auto-scalable not match, must delete and recreate")
}
}
checkDetailLB()
return inspect.serviceConf.LoadBalancerID, nil
Expand Down

0 comments on commit ea32779

Please sign in to comment.