diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 66ddb528b..bcf809d1e 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -51,6 +51,8 @@ func TestE2E(t *testing.T) { gomega.Expect(err).To(gomega.BeNil()) }) + gomega.RegisterFailHandler(ginkgo.Fail) + ginkgo.Describe("Run cloud controller manager e2e tests", func() { AddControllerTests(f) }) diff --git a/test/e2e/framework/expect.go b/test/e2e/framework/expect.go index c3541af16..c1dadad03 100644 --- a/test/e2e/framework/expect.go +++ b/test/e2e/framework/expect.go @@ -142,9 +142,20 @@ func loadBalancerAttrEqual(f *Framework, anno *annotation.AnnotationRequest, svc return fmt.Errorf("expected slb id %s, got %s", id, lb.LoadBalancerId) } } + instanceChargeType := anno.Get(annotation.InstanceChargeType) - if spec := anno.Get(annotation.Spec); spec != "" && string(lb.LoadBalancerSpec) != spec { - return fmt.Errorf("expected slb spec %s, got %s", spec, lb.LoadBalancerSpec) + if instanceChargeType != "" { + if instanceChargeType != string(lb.InstanceChargeType) { + return fmt.Errorf("expected slb instanceChargeType %s, got %s", instanceChargeType, lb.InstanceChargeType) + } + } else { + instanceChargeType = string(lb.InstanceChargeType) + } + if model.InstanceChargeType(instanceChargeType).IsPayBySpec() { + spec := anno.Get(annotation.Spec) + if spec != "" && string(lb.LoadBalancerSpec) != spec { + return fmt.Errorf("expected slb spec %s, got %s", spec, lb.LoadBalancerSpec) + } } if paymentType := anno.Get(annotation.ChargeType); paymentType != "" { diff --git a/test/e2e/framework/expect_nlb.go b/test/e2e/framework/expect_nlb.go index 5320e95dc..69e8a4716 100644 --- a/test/e2e/framework/expect_nlb.go +++ b/test/e2e/framework/expect_nlb.go @@ -300,6 +300,12 @@ func nlbVsgAttrEqual(f *Framework, reqCtx *svcCtx.RequestContext, remote *nlbmod sg.IsUserManaged = true } if found { + sgType := reqCtx.Anno.Get(annotation.ServerGroupType) + if sgType != "" && nlbmodel.ServerGroupType(sgType) != sg.ServerGroupType { + return fmt.Errorf("server group %s type not equal, local: %s, remote: %s", + sg.ServerGroupName, reqCtx.Anno.Get(annotation.ServerGroupType), sg.ServerGroupType) + } + sg.ServicePort = &port sg.ServicePort.Protocol = v1.Protocol(proto) if isOverride(reqCtx.Anno) && !isNLBServerGroupUsedByPort(sg, remote.Listeners) { @@ -444,36 +450,21 @@ func isNLBBackendEqual(client *client.KubeClient, reqCtx *svcCtx.RequestContext, for _, l := range backends { found := false for _, r := range sg.Servers { - if policy == helper.ENITrafficPolicy { - if l.ServerIp == r.ServerIp && - l.Port == r.Port && - l.ServerType == model.ENIBackendType { - if !sg.IsUserManaged && l.Description != r.Description { - return false, fmt.Errorf("mode %s expected vgroup [%s] backend %s description not equal,"+ - " expect %s, got %s", policy, sg.ServerGroupId, l.ServerIp, l.Description, r.Description) - } - if l.Weight != r.Weight { - return false, fmt.Errorf("mode %s expected vgroup [%s] backend %s weight not equal,"+ - " expect %d, got %d", policy, sg.ServerGroupId, l.ServerIp, l.Weight, r.Weight) - } - found = true - break + if isServerEqual(l, r) { + if l.Port != r.Port { + return false, fmt.Errorf("expected servergroup [%s] backend %s port not equal,"+ + " expect %d, got %d", sg.ServerGroupId, r.ServerId, l.Port, r.Port) } - } else { - if l.ServerId == r.ServerId && - l.Port == r.Port && - l.ServerType == model.ECSBackendType { - if !sg.IsUserManaged && l.Description != r.Description { - return false, fmt.Errorf("mode %s expected vgroup [%s] backend %s description not equal,"+ - " expect %s, got %s", policy, sg.ServerGroupId, l.ServerIp, l.Description, r.Description) - } - if l.Weight != r.Weight { - return false, fmt.Errorf("mode %s expected vgroup [%s] backend %s weight not equal,"+ - " expect %d, got %d", policy, sg.ServerGroupId, l.ServerIp, l.Weight, r.Weight) - } - found = true - break + if l.Weight != r.Weight { + return false, fmt.Errorf("expected servergroup [%s] backend %s weight not equal,"+ + " expect %d, got %d", sg.ServerGroupId, r.ServerId, l.Weight, r.Weight) + } + if l.Description != r.Description { + return false, fmt.Errorf("expected servergroup [%s] backend %s description not equal,"+ + " expect %s, got %s", sg.ServerGroupId, r.ServerId, l.Description, r.Description) } + found = true + break } } if !found { @@ -484,6 +475,24 @@ func isNLBBackendEqual(client *client.KubeClient, reqCtx *svcCtx.RequestContext, return true, nil } +func isServerEqual(a, b nlbmodel.ServerGroupServer) bool { + if a.ServerType != b.ServerType { + return false + } + + switch a.ServerType { + case nlbmodel.EniServerType: + return a.ServerId == b.ServerId && a.ServerIp == b.ServerIp + case nlbmodel.EcsServerType: + return a.ServerId == b.ServerId + case nlbmodel.IpServerType: + return a.ServerId == b.ServerId && a.ServerIp == b.ServerIp + default: + klog.Errorf("%s is not supported, skip", a.ServerType) + return false + } +} + func nlbTCPEqual(reqCtx *svcCtx.RequestContext, local v1.ServicePort, remote *nlbmodel.ListenerAttribute) error { if err := genericNLBServerEqual(reqCtx, local, remote); err != nil { return err @@ -589,10 +598,20 @@ func buildServerGroupENIBackends(anno *annotation.AnnotationRequest, ep *v1.Endp Description: sg.ServerGroupName, ServerIp: address.IP, Port: int32(backendPort), - ServerType: model.ENIBackendType, }) } } + + if sg.ServerGroupType == nlbmodel.IpServerGroupType { + for i := range ret { + ret[i].ServerId = ret[i].ServerIp + ret[i].ServerType = nlbmodel.IpServerType + } + } else { + for i := range ret { + ret[i].ServerType = model.ENIBackendType + } + } return setServerGroupWeightBackends(helper.ENITrafficPolicy, ret, sg.Weight), nil } @@ -615,12 +634,27 @@ func buildServerGroupLocalBackends(anno *annotation.AnnotationRequest, ep *v1.En if err != nil { return nil, fmt.Errorf("providerID %s parse error: %s", node.Spec.ProviderID, err.Error()) } - ret = append(ret, nlbmodel.ServerGroupServer{ - Description: sg.ServerGroupName, - ServerId: id, - Port: sg.ServicePort.NodePort, - ServerType: model.ECSBackendType, - }) + if sg.ServerGroupType == nlbmodel.IpServerGroupType { + ip, err := helper.GetNodeInternalIP(node) + if err != nil { + return nil, fmt.Errorf("get node address err: %s", err.Error()) + } + ret = append(ret, nlbmodel.ServerGroupServer{ + Description: sg.ServerGroupName, + ServerId: ip, + ServerIp: ip, + Port: sg.ServicePort.NodePort, + ServerType: nlbmodel.IpServerType, + }) + } else { + ret = append(ret, nlbmodel.ServerGroupServer{ + Description: sg.ServerGroupName, + ServerId: id, + Port: sg.ServicePort.NodePort, + ServerType: nlbmodel.EcsServerType, + }) + } + } } @@ -645,12 +679,22 @@ func buildServerGroupECIBackends(ep *v1.Endpoints, nodes []v1.Node, sg *nlbmodel } if isVKNode(*node) { backendPort := getBackendPort(*sg.ServicePort, subset) - ret = append(ret, nlbmodel.ServerGroupServer{ - Description: sg.ServerGroupName, - ServerIp: addr.IP, - Port: int32(backendPort), - ServerType: model.ENIBackendType, - }) + if sg.ServerGroupType == nlbmodel.IpServerGroupType { + ret = append(ret, nlbmodel.ServerGroupServer{ + Description: sg.ServerGroupName, + ServerId: addr.IP, + ServerIp: addr.IP, + Port: int32(backendPort), + ServerType: nlbmodel.IpServerType, + }) + } else { + ret = append(ret, nlbmodel.ServerGroupServer{ + Description: sg.ServerGroupName, + ServerIp: addr.IP, + Port: int32(backendPort), + ServerType: model.ENIBackendType, + }) + } } } } @@ -667,12 +711,28 @@ func buildServerGroupClusterBackends(anno *annotation.AnnotationRequest, ep *v1. if err != nil { return nil, fmt.Errorf("providerID %s parse error: %s", n.Spec.ProviderID, err.Error()) } - ret = append(ret, nlbmodel.ServerGroupServer{ - Description: sg.ServerGroupName, - ServerId: id, - ServerType: model.ECSBackendType, - Port: sg.ServicePort.NodePort, - }) + + if sg.ServerGroupType == nlbmodel.IpServerGroupType { + ip, err := helper.GetNodeInternalIP(&n) + if err != nil { + return nil, fmt.Errorf("get node address err: %s", err.Error()) + } + ret = append(ret, nlbmodel.ServerGroupServer{ + Description: sg.ServerGroupName, + ServerId: ip, + ServerIp: ip, + ServerType: nlbmodel.IpServerType, + Port: sg.ServicePort.NodePort, + }) + } else { + ret = append(ret, nlbmodel.ServerGroupServer{ + Description: sg.ServerGroupName, + ServerId: id, + ServerType: nlbmodel.EcsServerType, + Port: sg.ServicePort.NodePort, + }) + } + } eciBackends, err := buildServerGroupECIBackends(ep, nodes, sg) diff --git a/test/e2e/testcase/service/clbv1/loadbalancer.go b/test/e2e/testcase/service/clbv1/loadbalancer.go index 4b3a2848e..d87ffc404 100644 --- a/test/e2e/testcase/service/clbv1/loadbalancer.go +++ b/test/e2e/testcase/service/clbv1/loadbalancer.go @@ -625,14 +625,6 @@ func RunLoadBalancerTestCases(f *framework.Framework) { } ginkgo.Context("instance-charge-type", func() { - ginkgo.It("instance-charge-type: PayByCLCU", func() { - svc, err := f.Client.KubeClient.CreateServiceByAnno(map[string]string{ - annotation.Annotation(annotation.InstanceChargeType): "PayByCLCU", - }) - gomega.Expect(err).To(gomega.BeNil()) - err = f.ExpectLoadBalancerEqual(svc) - gomega.Expect(err).To(gomega.BeNil()) - }) ginkgo.It("instance-charge-type: PayBySpec -> PayByCLCU", func() { oldsvc, err := f.Client.KubeClient.CreateServiceByAnno(map[string]string{ annotation.Annotation(annotation.InstanceChargeType): "PayBySpec", @@ -681,37 +673,35 @@ func RunLoadBalancerTestCases(f *framework.Framework) { err = f.ExpectLoadBalancerEqual(newsvc) gomega.Expect(err).To(gomega.BeNil()) }) - ginkgo.It("instance-charge-type: PayBySpec & spec annotation", func() { + ginkgo.It("instance-charge-type: PayByCLCU & spec annotation", func() { oldsvc, err := f.Client.KubeClient.CreateServiceByAnno(map[string]string{ - annotation.Annotation(annotation.InstanceChargeType): "PayBySpec", - annotation.Annotation(annotation.Spec): model.S1Small, + annotation.Annotation(annotation.InstanceChargeType): "PayByCLCU", }) gomega.Expect(err).To(gomega.BeNil()) err = f.ExpectLoadBalancerEqual(oldsvc) gomega.Expect(err).To(gomega.BeNil()) newsvc := oldsvc.DeepCopy() - newsvc.Annotations[annotation.Annotation(annotation.Spec)] = "slb.s2.small" + newsvc.Annotations[annotation.Annotation(annotation.Spec)] = "slb.s1.small" newsvc, err = f.Client.KubeClient.PatchService(oldsvc, newsvc) gomega.Expect(err).To(gomega.BeNil()) err = f.ExpectLoadBalancerEqual(newsvc) gomega.Expect(err).To(gomega.BeNil()) }) - ginkgo.It("instance-charge-type: PayByCLCU & spec annotation", func() { + ginkgo.It("instance-charge-type: PayBySpec & spec annotation", func() { oldsvc, err := f.Client.KubeClient.CreateServiceByAnno(map[string]string{ - annotation.Annotation(annotation.InstanceChargeType): "PayByCLCU", - annotation.Annotation(annotation.Spec): model.S1Small, + annotation.Annotation(annotation.Spec): "slb.s2.small", }) gomega.Expect(err).To(gomega.BeNil()) err = f.ExpectLoadBalancerEqual(oldsvc) - gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err).To(gomega.BeNil()) newsvc := oldsvc.DeepCopy() - newsvc.Annotations[annotation.Annotation(annotation.Spec)] = "slb.s2.small" + newsvc.Annotations[annotation.Annotation(annotation.Spec)] = "slb.s1.small" newsvc, err = f.Client.KubeClient.PatchService(oldsvc, newsvc) gomega.Expect(err).To(gomega.BeNil()) err = f.ExpectLoadBalancerEqual(newsvc) - gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err).To(gomega.BeNil()) }) }) diff --git a/test/e2e/testcase/service/nlbv2/backend.go b/test/e2e/testcase/service/nlbv2/backend.go index db3b7b6ed..efd96123e 100644 --- a/test/e2e/testcase/service/nlbv2/backend.go +++ b/test/e2e/testcase/service/nlbv2/backend.go @@ -4,6 +4,7 @@ import ( "fmt" "k8s.io/cloud-provider-alibaba-cloud/pkg/controller/helper" "k8s.io/cloud-provider-alibaba-cloud/pkg/controller/service/reconcile/annotation" + "k8s.io/cloud-provider-alibaba-cloud/pkg/model/nlb" "time" "github.com/onsi/ginkgo/v2" @@ -652,5 +653,39 @@ func RunBackendTestCases(f *framework.Framework) { gomega.Expect(err).To(gomega.BeNil()) }) }) + + ginkgo.Context("server-group-type", func() { + ginkgo.It("ip", func() { + rawsvc := f.Client.KubeClient.DefaultNLBService() + rawsvc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal + rawsvc.Annotations = map[string]string{ + annotation.Annotation(annotation.ZoneMaps): options.TestConfig.NLBZoneMaps, + annotation.Annotation(annotation.ServerGroupType): string(nlb.IpServerGroupType), + } + svc, err := f.Client.KubeClient.CreateService(rawsvc) + gomega.Expect(err).To(gomega.BeNil()) + err = f.ExpectNetworkLoadBalancerEqual(svc) + gomega.Expect(err).To(gomega.BeNil()) + }) + ginkgo.It("ip -> instance", func() { + rawsvc := f.Client.KubeClient.DefaultNLBService() + rawsvc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal + rawsvc.Annotations = map[string]string{ + annotation.Annotation(annotation.ZoneMaps): options.TestConfig.NLBZoneMaps, + annotation.Annotation(annotation.ServerGroupType): string(nlb.IpServerGroupType), + } + oldsvc, err := f.Client.KubeClient.CreateService(rawsvc) + gomega.Expect(err).To(gomega.BeNil()) + err = f.ExpectNetworkLoadBalancerEqual(oldsvc) + gomega.Expect(err).To(gomega.BeNil()) + + newsvc := oldsvc.DeepCopy() + newsvc.Annotations[annotation.Annotation(annotation.ServerGroupType)] = string(nlb.InstanceServerGroupType) + newsvc, err = f.Client.KubeClient.PatchService(oldsvc, newsvc) + gomega.Expect(err).To(gomega.BeNil()) + err = f.ExpectNetworkLoadBalancerEqual(newsvc) + gomega.Expect(err).To(gomega.BeNil()) + }) + }) }) }