Merge pull request #313 from gujingit/feature/e2e

e2e tests
kubernetes · Jun 8, 2022 · 2bfcd7c · 2bfcd7c
2 parents 4b79437 + 6290319
commit 2bfcd7c
Show file tree

Hide file tree

Showing 14 changed files with 319 additions and 49 deletions.
diff --git a/pkg/controller/route/manager.go b/pkg/controller/route/manager.go
@@ -46,6 +46,7 @@ func createRouteForInstance(ctx context.Context, table, providerID, cidr string,
 					return true, nil
 				}
 				// fail fast, wait next time reconcile
+				klog.Errorf("Backoff creating route: same cidr with different providerID, %s", innerErr.Error())
 				return false, innerErr
 			}
 			klog.Errorf("Backoff creating route: %s", innerErr.Error())

diff --git a/pkg/controller/route/route_controller.go b/pkg/controller/route/route_controller.go
@@ -152,7 +152,7 @@ func (r *ReconcileRoute) Reconcile(ctx context.Context, request reconcile.Reques
 
 	err = r.syncCloudRoute(ctx, reconcileNode)
 	if err != nil {
-		klog.Errorf("sync cloud route failed, node: %s, err: %s", reconcileNode.Name, err.Error())
+		klog.Errorf("add route for node %s failed, err: %s", reconcileNode.Name, err.Error())
 		nodeRef := &corev1.ObjectReference{
 			Kind:      "Node",
 			Name:      reconcileNode.Name,
@@ -305,6 +305,9 @@ func (r *ReconcileRoute) periodicalSync() {
 func (r *ReconcileRoute) reconcileForCluster() {
 	ctx := context.Background()
 	start := time.Now()
+	defer func() {
+		metric.RouteLatency.WithLabelValues("reconcile").Observe(metric.MsSince(start))
+	}()
 
 	nodes, err := node.NodeList(r.client)
 	if err != nil {
@@ -320,20 +323,25 @@ func (r *ReconcileRoute) reconcileForCluster() {
 			corev1.EventTypeWarning, helper.FailedSyncRoute,
 			fmt.Sprintf("Reconciling route error: %s", err.Error()),
 		)
+		return
 	}
 
+	var failedTableIds []string
 	for _, table := range tables {
-		// Sync for Nodes
-		err := r.syncTableRoutes(ctx, table, nodes)
-		if err != nil {
+		// Sync for nodes
+		if err := r.syncTableRoutes(ctx, table, nodes); err != nil {
+			failedTableIds = append(failedTableIds, table)
 			klog.Errorf("sync route tables error: sync table [%s] error: %s", table, err.Error())
-			r.record.Event(
-				&corev1.Event{ObjectMeta: metav1.ObjectMeta{Name: "route-controller"}},
-				corev1.EventTypeWarning, helper.FailedSyncRoute,
-				fmt.Sprintf("Error reconciling route, reconcile table [%s] failed", table),
-			)
 		}
 	}
-	metric.RouteLatency.WithLabelValues("reconcile").Observe(metric.MsSince(start))
-	klog.Infof("sync route tables successfully, tables [%s]", strings.Join(tables, ","))
+
+	if len(failedTableIds) != 0 {
+		r.record.Event(
+			&corev1.Event{ObjectMeta: metav1.ObjectMeta{Name: "route-controller"}},
+			corev1.EventTypeWarning, helper.FailedSyncRoute,
+			fmt.Sprintf("Error reconciling route, reconcile table [%s] failed", strings.Join(failedTableIds, ",")),
+		)
+	} else {
+		klog.Infof("sync route tables successfully, tables [%s]", strings.Join(tables, ","))
+	}
 }
diff --git a/pkg/controller/service/loadbalancer.go b/pkg/controller/service/loadbalancer.go
@@ -113,8 +113,7 @@ func (mgr *LoadBalancerManager) Update(reqCtx *RequestContext, local, remote *mo
 	}
 
 	// only supports pay by spec instances
-	if local.LoadBalancerAttribute.InstanceChargeType == model.PayBySpec ||
-		local.LoadBalancerAttribute.InstanceChargeType == "" {
+	if local.LoadBalancerAttribute.InstanceChargeType.IsPayBySpec() {
 		// update chargeType & bandwidth
 		needUpdate, charge, bandwidth := false, remote.LoadBalancerAttribute.InternetChargeType, remote.LoadBalancerAttribute.Bandwidth
 		if local.LoadBalancerAttribute.InternetChargeType != "" &&
@@ -216,14 +215,16 @@ func (mgr *LoadBalancerManager) BuildLocalModel(reqCtx *RequestContext, mdl *mod
 	mdl.LoadBalancerAttribute.AddressType = model.AddressType(reqCtx.Anno.Get(AddressType))
 	mdl.LoadBalancerAttribute.InternetChargeType = model.InternetChargeType(reqCtx.Anno.Get(ChargeType))
 	mdl.LoadBalancerAttribute.InstanceChargeType = model.InstanceChargeType(reqCtx.Anno.Get(InstanceChargeType))
-	bandwidth := reqCtx.Anno.Get(Bandwidth)
-	if bandwidth != "" {
-		i, err := strconv.Atoi(bandwidth)
-		if err != nil &&
-			mdl.LoadBalancerAttribute.InternetChargeType == model.PayByBandwidth {
-			return fmt.Errorf("bandwidth must be integer, got [%s], error: %s", bandwidth, err.Error())
+	if mdl.LoadBalancerAttribute.InstanceChargeType.IsPayBySpec() {
+		bandwidth := reqCtx.Anno.Get(Bandwidth)
+		if bandwidth != "" {
+			i, err := strconv.Atoi(bandwidth)
+			if err != nil {
+				return fmt.Errorf("bandwidth must be integer, got [%s], error: %s", bandwidth, err.Error())
+			}
+			mdl.LoadBalancerAttribute.Bandwidth = i
 		}
-		mdl.LoadBalancerAttribute.Bandwidth = i
+		mdl.LoadBalancerAttribute.LoadBalancerSpec = model.LoadBalancerSpecType(reqCtx.Anno.Get(Spec))
 	}
 	if reqCtx.Anno.Get(LoadBalancerId) != "" {
 		mdl.LoadBalancerAttribute.LoadBalancerId = reqCtx.Anno.Get(LoadBalancerId)
@@ -233,7 +234,6 @@ func (mgr *LoadBalancerManager) BuildLocalModel(reqCtx *RequestContext, mdl *mod
 	mdl.LoadBalancerAttribute.VSwitchId = reqCtx.Anno.Get(VswitchId)
 	mdl.LoadBalancerAttribute.MasterZoneId = reqCtx.Anno.Get(MasterZoneID)
 	mdl.LoadBalancerAttribute.SlaveZoneId = reqCtx.Anno.Get(SlaveZoneID)
-	mdl.LoadBalancerAttribute.LoadBalancerSpec = model.LoadBalancerSpecType(reqCtx.Anno.Get(Spec))
 	mdl.LoadBalancerAttribute.ResourceGroupId = reqCtx.Anno.Get(ResourceGroupId)
 	mdl.LoadBalancerAttribute.AddressIPVersion = model.AddressIPVersionType(reqCtx.Anno.Get(IPVersion))
 	mdl.LoadBalancerAttribute.DeleteProtection = model.FlagType(reqCtx.Anno.Get(DeleteProtection))
@@ -282,7 +282,8 @@ func setModelDefaultValue(mgr *LoadBalancerManager, mdl *model.LoadBalancer, ann
 		}
 	}
 
-	if mdl.LoadBalancerAttribute.LoadBalancerSpec == "" {
+	if mdl.LoadBalancerAttribute.InstanceChargeType.IsPayBySpec() &&
+		mdl.LoadBalancerAttribute.LoadBalancerSpec == "" {
 		mdl.LoadBalancerAttribute.LoadBalancerSpec = model.LoadBalancerSpecType(anno.GetDefaultValue(Spec))
 	}
 

diff --git a/pkg/controller/service/service_controller.go b/pkg/controller/service/service_controller.go
@@ -322,7 +322,7 @@ func (m *ReconcileService) buildAndApplyModel(reqCtx *RequestContext) (*model.Lo
 	// apply model
 	remoteModel, err := m.applier.Apply(reqCtx, localModel)
 	if err != nil {
-		return nil, fmt.Errorf("apply model error: %s", err.Error())
+		return remoteModel, fmt.Errorf("apply model error: %s", err.Error())
 	}
 	return remoteModel, nil
 }

diff --git a/pkg/controller/service/vgroups.go b/pkg/controller/service/vgroups.go
@@ -431,8 +431,8 @@ func needExcludeFromLB(reqCtx *RequestContext, node *v1.Node) bool {
 	// Remove nodes that are about to be deleted by the cluster autoscaler.
 	for _, taint := range node.Spec.Taints {
 		if taint.Key == ToBeDeletedTaint {
-			klog.V(4).Infof("Ignoring node %v with autoscaler taint %+v", node.Name, taint)
-			return false
+			klog.Infof("Ignoring node %v with autoscaler taint %+v", node.Name, taint)
+			return true
 		}
 	}
 

diff --git a/pkg/model/load_balancer.go b/pkg/model/load_balancer.go
@@ -27,7 +27,9 @@ const PayByBandwidth = InternetChargeType("paybybandwidth")
 // InstanceChargeType slb instance charge type
 type InstanceChargeType string
 
-const PayBySpec = InstanceChargeType("PayBySpec")
+func (t InstanceChargeType) IsPayBySpec() bool {
+	return t == "" || strings.ToLower(string(t)) == "paybyspec"
+}
 
 type AddressIPVersionType string
 

diff --git a/pkg/provider/alibaba/slb/loadbalancer.go b/pkg/provider/alibaba/slb/loadbalancer.go
@@ -316,6 +316,20 @@ func (p SLBProvider) DescribeServerCertificates(ctx context.Context) ([]string,
 	return certs, nil
 }
 
+// DescribeCACertificates used for e2etest
+func (p SLBProvider) DescribeCACertificates(ctx context.Context) ([]string, error) {
+	req := slb.CreateDescribeCACertificatesRequest()
+	resp, err := p.auth.SLB.DescribeCACertificates(req)
+	if err != nil {
+		return nil, err
+	}
+	var certIds []string
+	for _, cert := range resp.CACertificates.CACertificate {
+		certIds = append(certIds, cert.CACertificateId)
+	}
+	return certIds, nil
+}
+
 func setRequest(request *slb.CreateLoadBalancerRequest, mdl *model.LoadBalancer) {
 	if mdl.LoadBalancerAttribute.AddressType != "" {
 		request.AddressType = string(mdl.LoadBalancerAttribute.AddressType)
@@ -341,6 +355,9 @@ func setRequest(request *slb.CreateLoadBalancerRequest, mdl *model.LoadBalancer)
 	if mdl.LoadBalancerAttribute.SlaveZoneId != "" {
 		request.SlaveZoneId = mdl.LoadBalancerAttribute.SlaveZoneId
 	}
+	if mdl.LoadBalancerAttribute.InstanceChargeType != "" {
+		request.InstanceChargeType = string(mdl.LoadBalancerAttribute.InstanceChargeType)
+	}
 	if mdl.LoadBalancerAttribute.LoadBalancerSpec != "" {
 		request.LoadBalancerSpec = string(mdl.LoadBalancerAttribute.LoadBalancerSpec)
 	}

diff --git a/test/e2e/client/client.go b/test/e2e/client/client.go
@@ -202,5 +202,15 @@ func (client *E2EClient) InitOptions() error {
 			}
 		}
 	}
+
+	if options.TestConfig.CACertID == "" {
+		cacerts, err := client.CloudClient.DescribeCACertificates(context.TODO())
+		if err != nil {
+			return fmt.Errorf("DescribeCACertificates error: %s", err.Error())
+		}
+		if len(cacerts) > 0 {
+			options.TestConfig.CACertID = cacerts[0]
+		}
+	}
 	return nil
 }
diff --git a/test/e2e/client/kubeclient.go b/test/e2e/client/kubeclient.go
@@ -460,6 +460,9 @@ func (client *KubeClient) UnLabelNode(nodeName string, key string) error {
 func (client *KubeClient) UnscheduledNode(nodeName string) error {
 	return wait.PollImmediate(2*time.Second, time.Minute, func() (done bool, err error) {
 		n, err := client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
+		if err != nil || n == nil {
+			return false, nil
+		}
 		n.Spec.Unschedulable = true
 		_, err = client.CoreV1().Nodes().Update(context.TODO(), n, metav1.UpdateOptions{})
 		if err != nil {
@@ -473,6 +476,9 @@ func (client *KubeClient) UnscheduledNode(nodeName string) error {
 func (client *KubeClient) ScheduledNode(nodeName string) error {
 	return wait.PollImmediate(2*time.Second, time.Minute, func() (done bool, err error) {
 		n, err := client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
+		if err != nil || n == nil {
+			return false, nil
+		}
 		n.Spec.Unschedulable = false
 		_, err = client.CoreV1().Nodes().Update(context.TODO(), n, metav1.UpdateOptions{})
 		if err != nil {
@@ -483,6 +489,47 @@ func (client *KubeClient) ScheduledNode(nodeName string) error {
 
 }
 
+func (client *KubeClient) AddTaint(nodeName string, taint v1.Taint) error {
+	return wait.PollImmediate(2*time.Second, 30*time.Second, func() (done bool, err error) {
+		n, err := client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
+		if err != nil {
+			return false, nil
+		}
+		for _, taint := range n.Spec.Taints {
+			if taint.Key == taint.Key {
+				return true, nil
+			}
+		}
+		n.Spec.Taints = append(n.Spec.Taints, taint)
+		_, err = client.CoreV1().Nodes().Update(context.TODO(), n, metav1.UpdateOptions{})
+		if err != nil {
+			return false, nil
+		}
+		return true, nil
+	})
+}
+
+func (client *KubeClient) RemoveTaint(nodeName string, taint v1.Taint) error {
+	return wait.PollImmediate(2*time.Second, 30*time.Second, func() (done bool, err error) {
+		n, err := client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
+		if err != nil {
+			return false, nil
+		}
+		var updateTaints []v1.Taint
+		for _, t := range n.Spec.Taints {
+			if t.Key == taint.Key {
+				continue
+			}
+			updateTaints = append(updateTaints, t)
+		}
+		_, err = client.CoreV1().Nodes().Update(context.TODO(), n, metav1.UpdateOptions{})
+		if err != nil {
+			return false, nil
+		}
+		return true, nil
+	})
+}
+
 func (client *KubeClient) ListNodes() ([]v1.Node, error) {
 	nodeList, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
 	if err != nil {
@@ -505,9 +552,6 @@ func (client *KubeClient) GetLatestNode() (*v1.Node, error) {
 		if helper.HasExcludeLabel(&node) {
 			continue
 		}
-		if _, isMaster := node.Labels[service.LabelNodeRoleMaster]; isMaster {
-			continue
-		}
 		if _, exclude := node.Labels[service.LabelNodeExcludeBalancer]; exclude {
 			continue
 		}