add e2etests

pkg/controller/service/loadbalancer.go

@@ -219,13 +219,11 @@ func (mgr *LoadBalancerManager) BuildLocalModel(reqCtx *RequestContext, mdl *mod
 		bandwidth := reqCtx.Anno.Get(Bandwidth)
 		if bandwidth != "" {
 			i, err := strconv.Atoi(bandwidth)
-			if err != nil &&
-				mdl.LoadBalancerAttribute.InternetChargeType == model.PayByBandwidth {
+			if err != nil {
 				return fmt.Errorf("bandwidth must be integer, got [%s], error: %s", bandwidth, err.Error())
 			}
 			mdl.LoadBalancerAttribute.Bandwidth = i
 		}
-
 		mdl.LoadBalancerAttribute.LoadBalancerSpec = model.LoadBalancerSpecType(reqCtx.Anno.Get(Spec))
 	}
 	if reqCtx.Anno.Get(LoadBalancerId) != "" {

test/e2e/client/client.go

@@ -202,5 +202,15 @@ func (client *E2EClient) InitOptions() error {
 			}
 		}
 	}
+
+	if options.TestConfig.CACertID == "" {
+		cacerts, err := client.CloudClient.DescribeCACertificates(context.TODO())
+		if err != nil {
+			return fmt.Errorf("DescribeCACertificates error: %s", err.Error())
+		}
+		if len(cacerts) > 0 {
+			options.TestConfig.CACertID = cacerts[0]
+		}
+	}
 	return nil
 }

test/e2e/client/kubeclient.go

@@ -460,6 +460,9 @@ func (client *KubeClient) UnLabelNode(nodeName string, key string) error {
 func (client *KubeClient) UnscheduledNode(nodeName string) error {
 	return wait.PollImmediate(2*time.Second, time.Minute, func() (done bool, err error) {
 		n, err := client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
+		if err != nil || n == nil {
+			return false, nil
+		}
 		n.Spec.Unschedulable = true
 		_, err = client.CoreV1().Nodes().Update(context.TODO(), n, metav1.UpdateOptions{})
 		if err != nil {
@@ -473,6 +476,9 @@ func (client *KubeClient) UnscheduledNode(nodeName string) error {
 func (client *KubeClient) ScheduledNode(nodeName string) error {
 	return wait.PollImmediate(2*time.Second, time.Minute, func() (done bool, err error) {
 		n, err := client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
+		if err != nil || n == nil {
+			return false, nil
+		}
 		n.Spec.Unschedulable = false
 		_, err = client.CoreV1().Nodes().Update(context.TODO(), n, metav1.UpdateOptions{})
 		if err != nil {
@@ -483,6 +489,47 @@ func (client *KubeClient) ScheduledNode(nodeName string) error {
 
 }
 
+func (client *KubeClient) AddTaint(nodeName string, taint v1.Taint) error {
+	return wait.PollImmediate(2*time.Second, 30*time.Second, func() (done bool, err error) {
+		n, err := client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
+		if err != nil {
+			return false, nil
+		}
+		for _, taint := range n.Spec.Taints {
+			if taint.Key == taint.Key {
+				return true, nil
+			}
+		}
+		n.Spec.Taints = append(n.Spec.Taints, taint)
+		_, err = client.CoreV1().Nodes().Update(context.TODO(), n, metav1.UpdateOptions{})
+		if err != nil {
+			return false, nil
+		}
+		return true, nil
+	})
+}
+
+func (client *KubeClient) RemoveTaint(nodeName string, taint v1.Taint) error {
+	return wait.PollImmediate(2*time.Second, 30*time.Second, func() (done bool, err error) {
+		n, err := client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
+		if err != nil {
+			return false, nil
+		}
+		var updateTaints []v1.Taint
+		for _, t := range n.Spec.Taints {
+			if t.Key == taint.Key {
+				continue
+			}
+			updateTaints = append(updateTaints, t)
+		}
+		_, err = client.CoreV1().Nodes().Update(context.TODO(), n, metav1.UpdateOptions{})
+		if err != nil {
+			return false, nil
+		}
+		return true, nil
+	})
+}
+
 func (client *KubeClient) ListNodes() ([]v1.Node, error) {
 	nodeList, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
 	if err != nil {
@@ -505,9 +552,6 @@ func (client *KubeClient) GetLatestNode() (*v1.Node, error) {
 		if helper.HasExcludeLabel(&node) {
 			continue
 		}
-		if _, isMaster := node.Labels[service.LabelNodeRoleMaster]; isMaster {
-			continue
-		}
 		if _, exclude := node.Labels[service.LabelNodeExcludeBalancer]; exclude {
 			continue
 		}

test/e2e/framework/expect.go

@@ -106,6 +106,28 @@ func (f *Framework) ExpectLoadBalancerClean(svc *v1.Service, remote *model.LoadB
 	return nil
 }
 
+func (f *Framework) ExpectLoadBalancerDeleted(svc *v1.Service) error {
+	reqCtx := &service.RequestContext{
+		Service: svc,
+		Anno:    service.NewAnnotationRequest(svc),
+	}
+	lbManager := service.NewLoadBalancerManager(f.Client.CloudClient)
+
+	return wait.PollImmediate(5*time.Second, 30*time.Second, func() (done bool, err error) {
+		lbMdl := &model.LoadBalancer{
+			NamespacedName: util.NamespacedName(svc),
+		}
+		err = lbManager.Find(reqCtx, lbMdl)
+		if err != nil {
+			return false, err
+		}
+		if lbMdl.LoadBalancerAttribute.LoadBalancerId != "" {
+			return false, nil
+		}
+		return true, nil
+	})
+}
+
 func isOverride(anno *service.AnnotationRequest) bool {
 	return anno.Get(service.LoadBalancerId) != "" && anno.Get(service.OverrideListener) == "true"
 }
@@ -116,29 +138,33 @@ func loadBalancerAttrEqual(f *Framework, anno *service.AnnotationRequest, svc *v
 			return fmt.Errorf("expected slb id %s, got %s", id, lb.LoadBalancerId)
 		}
 	}
-	if spec := anno.Get(service.Spec); spec != "" {
-		if string(lb.LoadBalancerSpec) != spec {
+
+	if model.InstanceChargeType(anno.Get(service.InstanceChargeType)).IsPayBySpec() {
+		if spec := anno.Get(service.Spec); spec != "" && string(lb.LoadBalancerSpec) != spec {
 			return fmt.Errorf("expected slb spec %s, got %s", spec, lb.LoadBalancerSpec)
 		}
+
+		if paymentType := anno.Get(service.ChargeType); paymentType != "" {
+			klog.Infof("in, chargeType: svc %s, lb %s", paymentType, lb.InternetChargeType)
+			if string(lb.InternetChargeType) != paymentType {
+				return fmt.Errorf("expected slb payment %s, got %s", paymentType, lb.InternetChargeType)
+			}
+			if paymentType == string(model.PayByBandwidth) {
+				if Bandwidth := anno.Get(service.Bandwidth); Bandwidth != "" {
+					if strconv.Itoa(lb.Bandwidth) != Bandwidth {
+						return fmt.Errorf("expected slb Bandwidth %s, got %d", Bandwidth, lb.Bandwidth)
+					}
+				}
+			}
+		}
 	}
+
 	if AddressType := anno.Get(service.AddressType); AddressType != "" {
 		if string(lb.AddressType) != AddressType {
 			return fmt.Errorf("expected slb AddressType %s, got %s", AddressType, lb.AddressType)
 		}
 	}
-	if paymentType := anno.Get(service.ChargeType); paymentType != "" {
-		klog.Infof("in, chargeType: svc %s, lb %s", paymentType, lb.InternetChargeType)
-		if string(lb.InternetChargeType) != paymentType {
-			return fmt.Errorf("expected slb payment %s, got %s", paymentType, lb.InternetChargeType)
-		}
-		if paymentType == string(model.PayByBandwidth) {
-			if Bandwidth := anno.Get(service.Bandwidth); Bandwidth != "" {
-				if strconv.Itoa(lb.Bandwidth) != Bandwidth {
-					return fmt.Errorf("expected slb Bandwidth %s, got %d", Bandwidth, lb.Bandwidth)
-				}
-			}
-		}
-	}
+
 	if LoadBalancerName := anno.Get(service.LoadBalancerName); LoadBalancerName != "" {
 		if lb.LoadBalancerName != LoadBalancerName {
 			return fmt.Errorf("expected slb name %s, got %s", LoadBalancerName, lb.LoadBalancerName)
@@ -727,6 +753,11 @@ func httpsEqual(reqCtx *service.RequestContext, local v1.ServicePort, remote mod
 			return fmt.Errorf("expected slb requestTimeout %d, got %d", timeout, remote.RequestTimeout)
 		}
 	}
+	if tls := reqCtx.Anno.Get(service.TLSCipherPolicy); tls != "" {
+		if remote.TLSCipherPolicy != tls {
+			return fmt.Errorf("expected slb tls %d, got %d", tls, remote.TLSCipherPolicy)
+		}
+	}
 	return nil
 }
 
@@ -1145,6 +1176,12 @@ func isNodeExcludeFromLoadBalancer(node *v1.Node, anno *service.AnnotationReques
 		return true
 	}
 
+	for _, taint := range node.Spec.Taints {
+		if taint.Key == service.ToBeDeletedTaint {
+			return true
+		}
+	}
+
 	if _, exclude := node.Labels[service.LabelNodeExcludeBalancer]; exclude {
 		return true
 	}
@@ -1283,6 +1320,37 @@ func (f *Framework) DeleteRouteEntry(node *v1.Node) error {
 	return nil
 }
 
+func (f *Framework) AddRouteEntry(prvdId, cidr string) error {
+	tables, err := f.Client.CloudClient.ListRouteTables(context.TODO(), options.TestConfig.VPCID)
+	if err != nil {
+		return err
+	}
+
+	for _, t := range tables {
+		var createErr error
+		err := wait.PollImmediate(5*time.Second, 20*time.Second, func() (done bool, err error) {
+			_, createErr = f.Client.CloudClient.CreateRoute(context.TODO(), t, prvdId, cidr)
+			if createErr != nil {
+				if strings.Contains(createErr.Error(), "InvalidCIDRBlock.Duplicate") {
+					route, findErr := f.Client.CloudClient.FindRoute(context.TODO(), t, prvdId, cidr)
+					if findErr == nil && route != nil {
+						return true, nil
+					}
+					// fail fast, wait next time reconcile
+					return false, findErr
+				}
+				klog.Errorf("Backoff creating route: %s", createErr.Error())
+				return false, nil
+			}
+			return true, nil
+		})
+		if err != nil {
+			return fmt.Errorf("add route entry error: %s, create error: %v", err.Error(), createErr)
+		}
+	}
+	return nil
+}
+
 func (f *Framework) ExpectRouteEqual() error {
 	var (
 		tables []string

test/e2e/options/options.go

@@ -35,6 +35,7 @@ type E2EConfig struct {
 	SlaveZoneID  string `json:"SlaveZoneID"`
 	CertID       string `json:"CertID"`
 	CertID2      string `json:"CertID2"`
+	CACertID     string `json:"CACertID"`
 
 	// created automatically if AllowCreateCloudResource=true
 	InternetLoadBalancerID string `json:"InternetLoadBalancerID"`
@@ -66,6 +67,7 @@ func (e *E2EConfig) BindFlags() {
 	flag.StringVar(&e.VSwitchID2, "vswitch-id-2", "", "vsw-id")
 	flag.StringVar(&e.CertID, "cert-id", "", "cert id")
 	flag.StringVar(&e.CertID2, "cert-id-2", "", "cert id")
+	flag.StringVar(&e.CACertID, "ca-cert-id", "", "ca-cert id")
 	flag.StringVar(&e.VServerGroupID, "vserver-group-id", "", "vserver group id")
 	flag.StringVar(&e.VServerGroupID2, "vserver-group-id-2", "", "vserver group id")
 }

test/e2e/testcase/route/route.go

@@ -36,6 +36,23 @@ func RunRouteControllerTestCases(f *framework.Framework) {
 			})
 		})
 
+		ginkgo.Context("conflict route entry", func() {
+			ginkgo.It("conflict route entry", func() {
+				nodes, err := f.Client.KubeClient.ListNodes()
+				gomega.Expect(err).To(gomega.BeNil())
+				if len(nodes) > 1 {
+					err = f.DeleteRouteEntry(&nodes[0])
+					gomega.Expect(err).To(gomega.BeNil())
+
+					err = f.AddRouteEntry(nodes[1].Spec.ProviderID, nodes[0].Spec.PodCIDR)
+					gomega.Expect(err).To(gomega.BeNil())
+				}
+
+				err = f.ExpectRouteEqual()
+				gomega.Expect(err).To(gomega.BeNil())
+			})
+		})
+
 		ginkgo.Context("multi vpc tables", func() {
 			ginkgo.It("multi vpc tables", func() {
 				// create route table

test/e2e/testcase/service/backend.go

@@ -2,6 +2,7 @@ package service
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/onsi/ginkgo"
 	"github.com/onsi/gomega"
@@ -140,6 +141,33 @@ func RunBackendTestCases(f *framework.Framework) {
 			})
 		})
 
+		ginkgo.Context("to-be-deleted-taint", func() {
+			ginkgo.It("node: to-be-deleted-taint", func() {
+				taint := v1.Taint{
+					Key:    service.ToBeDeletedTaint,
+					Value:  fmt.Sprint(time.Now().Unix()),
+					Effect: v1.TaintEffectNoSchedule,
+				}
+				// add ToBeDeletedTaint
+				node, err := f.Client.KubeClient.GetLatestNode()
+				gomega.Expect(err).To(gomega.BeNil())
+				gomega.Expect(node).NotTo(gomega.BeNil())
+				err = f.Client.KubeClient.AddTaint(node.Name, taint)
+				gomega.Expect(err).To(gomega.BeNil())
+
+				oldSvc, err := f.Client.KubeClient.CreateServiceByAnno(nil)
+				gomega.Expect(err).To(gomega.BeNil())
+				err = f.ExpectLoadBalancerEqual(oldSvc)
+				gomega.Expect(err).To(gomega.BeNil())
+
+				err = f.Client.KubeClient.RemoveTaint(node.Name, taint)
+				gomega.Expect(err).To(gomega.BeNil())
+				err = f.ExpectLoadBalancerEqual(oldSvc)
+				gomega.Expect(err).To(gomega.BeNil())
+			})
+
+		})
+
 		if options.TestConfig.Network == options.Terway {
 			ginkgo.Context("backend-type", func() {
 				ginkgo.It("backend-type: eni", func() {