Skip to content

Commit

Permalink
Merge pull request #384 from gujingit/test/e2etest
Browse files Browse the repository at this point in the history 
add nlb ip tests; update expect to support paybyclcu & serverGroupTDype
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot authored Oct 11, 2023
2 parents 60c4ae4 + d343ee8 commit 0c9637b
Show file tree
Hide file tree
Showing 5 changed files with 165 additions and 67 deletions.
2 changes: 2 additions & 0 deletions test/e2e/e2e_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ func TestE2E(t *testing.T) {
gomega.Expect(err).To(gomega.BeNil())
})

gomega.RegisterFailHandler(ginkgo.Fail)

ginkgo.Describe("Run cloud controller manager e2e tests", func() {
AddControllerTests(f)
})
Expand Down
15 changes: 13 additions & 2 deletions test/e2e/framework/expect.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,9 +142,20 @@ func loadBalancerAttrEqual(f *Framework, anno *annotation.AnnotationRequest, svc
return fmt.Errorf("expected slb id %s, got %s", id, lb.LoadBalancerId)
}
}
instanceChargeType := anno.Get(annotation.InstanceChargeType)

if spec := anno.Get(annotation.Spec); spec != "" && string(lb.LoadBalancerSpec) != spec {
return fmt.Errorf("expected slb spec %s, got %s", spec, lb.LoadBalancerSpec)
if instanceChargeType != "" {
if instanceChargeType != string(lb.InstanceChargeType) {
return fmt.Errorf("expected slb instanceChargeType %s, got %s", instanceChargeType, lb.InstanceChargeType)
}
} else {
instanceChargeType = string(lb.InstanceChargeType)
}
if model.InstanceChargeType(instanceChargeType).IsPayBySpec() {
spec := anno.Get(annotation.Spec)
if spec != "" && string(lb.LoadBalancerSpec) != spec {
return fmt.Errorf("expected slb spec %s, got %s", spec, lb.LoadBalancerSpec)
}
}

if paymentType := anno.Get(annotation.ChargeType); paymentType != "" {
Expand Down
154 changes: 107 additions & 47 deletions test/e2e/framework/expect_nlb.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -300,6 +300,12 @@ func nlbVsgAttrEqual(f *Framework, reqCtx *svcCtx.RequestContext, remote *nlbmod
sg.IsUserManaged = true
}
if found {
sgType := reqCtx.Anno.Get(annotation.ServerGroupType)
if sgType != "" && nlbmodel.ServerGroupType(sgType) != sg.ServerGroupType {
return fmt.Errorf("server group %s type not equal, local: %s, remote: %s",
sg.ServerGroupName, reqCtx.Anno.Get(annotation.ServerGroupType), sg.ServerGroupType)
}

sg.ServicePort = &port
sg.ServicePort.Protocol = v1.Protocol(proto)
if isOverride(reqCtx.Anno) && !isNLBServerGroupUsedByPort(sg, remote.Listeners) {
Expand Down Expand Up @@ -444,36 +450,21 @@ func isNLBBackendEqual(client *client.KubeClient, reqCtx *svcCtx.RequestContext,
for _, l := range backends {
found := false
for _, r := range sg.Servers {
if policy == helper.ENITrafficPolicy {
if l.ServerIp == r.ServerIp &&
l.Port == r.Port &&
l.ServerType == model.ENIBackendType {
if !sg.IsUserManaged && l.Description != r.Description {
return false, fmt.Errorf("mode %s expected vgroup [%s] backend %s description not equal,"+
" expect %s, got %s", policy, sg.ServerGroupId, l.ServerIp, l.Description, r.Description)
}
if l.Weight != r.Weight {
return false, fmt.Errorf("mode %s expected vgroup [%s] backend %s weight not equal,"+
" expect %d, got %d", policy, sg.ServerGroupId, l.ServerIp, l.Weight, r.Weight)
}
found = true
break
if isServerEqual(l, r) {
if l.Port != r.Port {
return false, fmt.Errorf("expected servergroup [%s] backend %s port not equal,"+
" expect %d, got %d", sg.ServerGroupId, r.ServerId, l.Port, r.Port)
}
} else {
if l.ServerId == r.ServerId &&
l.Port == r.Port &&
l.ServerType == model.ECSBackendType {
if !sg.IsUserManaged && l.Description != r.Description {
return false, fmt.Errorf("mode %s expected vgroup [%s] backend %s description not equal,"+
" expect %s, got %s", policy, sg.ServerGroupId, l.ServerIp, l.Description, r.Description)
}
if l.Weight != r.Weight {
return false, fmt.Errorf("mode %s expected vgroup [%s] backend %s weight not equal,"+
" expect %d, got %d", policy, sg.ServerGroupId, l.ServerIp, l.Weight, r.Weight)
}
found = true
break
if l.Weight != r.Weight {
return false, fmt.Errorf("expected servergroup [%s] backend %s weight not equal,"+
" expect %d, got %d", sg.ServerGroupId, r.ServerId, l.Weight, r.Weight)
}
if l.Description != r.Description {
return false, fmt.Errorf("expected servergroup [%s] backend %s description not equal,"+
" expect %s, got %s", sg.ServerGroupId, r.ServerId, l.Description, r.Description)
}
found = true
break
}
}
if !found {
Expand All @@ -484,6 +475,24 @@ func isNLBBackendEqual(client *client.KubeClient, reqCtx *svcCtx.RequestContext,
return true, nil
}

func isServerEqual(a, b nlbmodel.ServerGroupServer) bool {
if a.ServerType != b.ServerType {
return false
}

switch a.ServerType {
case nlbmodel.EniServerType:
return a.ServerId == b.ServerId && a.ServerIp == b.ServerIp
case nlbmodel.EcsServerType:
return a.ServerId == b.ServerId
case nlbmodel.IpServerType:
return a.ServerId == b.ServerId && a.ServerIp == b.ServerIp
default:
klog.Errorf("%s is not supported, skip", a.ServerType)
return false
}
}

func nlbTCPEqual(reqCtx *svcCtx.RequestContext, local v1.ServicePort, remote *nlbmodel.ListenerAttribute) error {
if err := genericNLBServerEqual(reqCtx, local, remote); err != nil {
return err
Expand Down Expand Up @@ -589,10 +598,20 @@ func buildServerGroupENIBackends(anno *annotation.AnnotationRequest, ep *v1.Endp
Description: sg.ServerGroupName,
ServerIp: address.IP,
Port: int32(backendPort),
ServerType: model.ENIBackendType,
})
}
}

if sg.ServerGroupType == nlbmodel.IpServerGroupType {
for i := range ret {
ret[i].ServerId = ret[i].ServerIp
ret[i].ServerType = nlbmodel.IpServerType
}
} else {
for i := range ret {
ret[i].ServerType = model.ENIBackendType
}
}
return setServerGroupWeightBackends(helper.ENITrafficPolicy, ret, sg.Weight), nil
}

Expand All @@ -615,12 +634,27 @@ func buildServerGroupLocalBackends(anno *annotation.AnnotationRequest, ep *v1.En
if err != nil {
return nil, fmt.Errorf("providerID %s parse error: %s", node.Spec.ProviderID, err.Error())
}
ret = append(ret, nlbmodel.ServerGroupServer{
Description: sg.ServerGroupName,
ServerId: id,
Port: sg.ServicePort.NodePort,
ServerType: model.ECSBackendType,
})
if sg.ServerGroupType == nlbmodel.IpServerGroupType {
ip, err := helper.GetNodeInternalIP(node)
if err != nil {
return nil, fmt.Errorf("get node address err: %s", err.Error())
}
ret = append(ret, nlbmodel.ServerGroupServer{
Description: sg.ServerGroupName,
ServerId: ip,
ServerIp: ip,
Port: sg.ServicePort.NodePort,
ServerType: nlbmodel.IpServerType,
})
} else {
ret = append(ret, nlbmodel.ServerGroupServer{
Description: sg.ServerGroupName,
ServerId: id,
Port: sg.ServicePort.NodePort,
ServerType: nlbmodel.EcsServerType,
})
}

}
}

Expand All @@ -645,12 +679,22 @@ func buildServerGroupECIBackends(ep *v1.Endpoints, nodes []v1.Node, sg *nlbmodel
}
if isVKNode(*node) {
backendPort := getBackendPort(*sg.ServicePort, subset)
ret = append(ret, nlbmodel.ServerGroupServer{
Description: sg.ServerGroupName,
ServerIp: addr.IP,
Port: int32(backendPort),
ServerType: model.ENIBackendType,
})
if sg.ServerGroupType == nlbmodel.IpServerGroupType {
ret = append(ret, nlbmodel.ServerGroupServer{
Description: sg.ServerGroupName,
ServerId: addr.IP,
ServerIp: addr.IP,
Port: int32(backendPort),
ServerType: nlbmodel.IpServerType,
})
} else {
ret = append(ret, nlbmodel.ServerGroupServer{
Description: sg.ServerGroupName,
ServerIp: addr.IP,
Port: int32(backendPort),
ServerType: model.ENIBackendType,
})
}
}
}
}
Expand All @@ -667,12 +711,28 @@ func buildServerGroupClusterBackends(anno *annotation.AnnotationRequest, ep *v1.
if err != nil {
return nil, fmt.Errorf("providerID %s parse error: %s", n.Spec.ProviderID, err.Error())
}
ret = append(ret, nlbmodel.ServerGroupServer{
Description: sg.ServerGroupName,
ServerId: id,
ServerType: model.ECSBackendType,
Port: sg.ServicePort.NodePort,
})

if sg.ServerGroupType == nlbmodel.IpServerGroupType {
ip, err := helper.GetNodeInternalIP(&n)
if err != nil {
return nil, fmt.Errorf("get node address err: %s", err.Error())
}
ret = append(ret, nlbmodel.ServerGroupServer{
Description: sg.ServerGroupName,
ServerId: ip,
ServerIp: ip,
ServerType: nlbmodel.IpServerType,
Port: sg.ServicePort.NodePort,
})
} else {
ret = append(ret, nlbmodel.ServerGroupServer{
Description: sg.ServerGroupName,
ServerId: id,
ServerType: nlbmodel.EcsServerType,
Port: sg.ServicePort.NodePort,
})
}

}

eciBackends, err := buildServerGroupECIBackends(ep, nodes, sg)
Expand Down
26 changes: 8 additions & 18 deletions test/e2e/testcase/service/clbv1/loadbalancer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -625,14 +625,6 @@ func RunLoadBalancerTestCases(f *framework.Framework) {
}

ginkgo.Context("instance-charge-type", func() {
ginkgo.It("instance-charge-type: PayByCLCU", func() {
svc, err := f.Client.KubeClient.CreateServiceByAnno(map[string]string{
annotation.Annotation(annotation.InstanceChargeType): "PayByCLCU",
})
gomega.Expect(err).To(gomega.BeNil())
err = f.ExpectLoadBalancerEqual(svc)
gomega.Expect(err).To(gomega.BeNil())
})
ginkgo.It("instance-charge-type: PayBySpec -> PayByCLCU", func() {
oldsvc, err := f.Client.KubeClient.CreateServiceByAnno(map[string]string{
annotation.Annotation(annotation.InstanceChargeType): "PayBySpec",
Expand Down Expand Up @@ -681,37 +673,35 @@ func RunLoadBalancerTestCases(f *framework.Framework) {
err = f.ExpectLoadBalancerEqual(newsvc)
gomega.Expect(err).To(gomega.BeNil())
})
ginkgo.It("instance-charge-type: PayBySpec & spec annotation", func() {
ginkgo.It("instance-charge-type: PayByCLCU & spec annotation", func() {
oldsvc, err := f.Client.KubeClient.CreateServiceByAnno(map[string]string{
annotation.Annotation(annotation.InstanceChargeType): "PayBySpec",
annotation.Annotation(annotation.Spec): model.S1Small,
annotation.Annotation(annotation.InstanceChargeType): "PayByCLCU",
})
gomega.Expect(err).To(gomega.BeNil())
err = f.ExpectLoadBalancerEqual(oldsvc)
gomega.Expect(err).To(gomega.BeNil())

newsvc := oldsvc.DeepCopy()
newsvc.Annotations[annotation.Annotation(annotation.Spec)] = "slb.s2.small"
newsvc.Annotations[annotation.Annotation(annotation.Spec)] = "slb.s1.small"
newsvc, err = f.Client.KubeClient.PatchService(oldsvc, newsvc)
gomega.Expect(err).To(gomega.BeNil())
err = f.ExpectLoadBalancerEqual(newsvc)
gomega.Expect(err).To(gomega.BeNil())
})
ginkgo.It("instance-charge-type: PayByCLCU & spec annotation", func() {
ginkgo.It("instance-charge-type: PayBySpec & spec annotation", func() {
oldsvc, err := f.Client.KubeClient.CreateServiceByAnno(map[string]string{
annotation.Annotation(annotation.InstanceChargeType): "PayByCLCU",
annotation.Annotation(annotation.Spec): model.S1Small,
annotation.Annotation(annotation.Spec): "slb.s2.small",
})
gomega.Expect(err).To(gomega.BeNil())
err = f.ExpectLoadBalancerEqual(oldsvc)
gomega.Expect(err).NotTo(gomega.BeNil())
gomega.Expect(err).To(gomega.BeNil())

newsvc := oldsvc.DeepCopy()
newsvc.Annotations[annotation.Annotation(annotation.Spec)] = "slb.s2.small"
newsvc.Annotations[annotation.Annotation(annotation.Spec)] = "slb.s1.small"
newsvc, err = f.Client.KubeClient.PatchService(oldsvc, newsvc)
gomega.Expect(err).To(gomega.BeNil())
err = f.ExpectLoadBalancerEqual(newsvc)
gomega.Expect(err).NotTo(gomega.BeNil())
gomega.Expect(err).To(gomega.BeNil())
})
})

Expand Down
35 changes: 35 additions & 0 deletions test/e2e/testcase/service/nlbv2/backend.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"k8s.io/cloud-provider-alibaba-cloud/pkg/controller/helper"
"k8s.io/cloud-provider-alibaba-cloud/pkg/controller/service/reconcile/annotation"
"k8s.io/cloud-provider-alibaba-cloud/pkg/model/nlb"
"time"

"github.com/onsi/ginkgo/v2"
Expand Down Expand Up @@ -652,5 +653,39 @@ func RunBackendTestCases(f *framework.Framework) {
gomega.Expect(err).To(gomega.BeNil())
})
})

ginkgo.Context("server-group-type", func() {
ginkgo.It("ip", func() {
rawsvc := f.Client.KubeClient.DefaultNLBService()
rawsvc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal
rawsvc.Annotations = map[string]string{
annotation.Annotation(annotation.ZoneMaps): options.TestConfig.NLBZoneMaps,
annotation.Annotation(annotation.ServerGroupType): string(nlb.IpServerGroupType),
}
svc, err := f.Client.KubeClient.CreateService(rawsvc)
gomega.Expect(err).To(gomega.BeNil())
err = f.ExpectNetworkLoadBalancerEqual(svc)
gomega.Expect(err).To(gomega.BeNil())
})
ginkgo.It("ip -> instance", func() {
rawsvc := f.Client.KubeClient.DefaultNLBService()
rawsvc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal
rawsvc.Annotations = map[string]string{
annotation.Annotation(annotation.ZoneMaps): options.TestConfig.NLBZoneMaps,
annotation.Annotation(annotation.ServerGroupType): string(nlb.IpServerGroupType),
}
oldsvc, err := f.Client.KubeClient.CreateService(rawsvc)
gomega.Expect(err).To(gomega.BeNil())
err = f.ExpectNetworkLoadBalancerEqual(oldsvc)
gomega.Expect(err).To(gomega.BeNil())

newsvc := oldsvc.DeepCopy()
newsvc.Annotations[annotation.Annotation(annotation.ServerGroupType)] = string(nlb.InstanceServerGroupType)
newsvc, err = f.Client.KubeClient.PatchService(oldsvc, newsvc)
gomega.Expect(err).To(gomega.BeNil())
err = f.ExpectNetworkLoadBalancerEqual(newsvc)
gomega.Expect(err).To(gomega.BeNil())
})
})
})
}

0 comments on commit 0c9637b

Please sign in to comment.