From 9f7117533ea4bbc674be3f2fdb4d419b5e4bb178 Mon Sep 17 00:00:00 2001 From: David Cheung Date: Mon, 4 Nov 2024 22:58:28 +0000 Subject: [PATCH] Update endpoints for NEGs from non-default subnet. --- pkg/neg/syncers/transaction.go | 42 +++-- pkg/neg/syncers/transaction_test.go | 231 +++++++++++++++++++++++++++- 2 files changed, 255 insertions(+), 18 deletions(-) diff --git a/pkg/neg/syncers/transaction.go b/pkg/neg/syncers/transaction.go index 0e9fad7d9..d4f77e545 100644 --- a/pkg/neg/syncers/transaction.go +++ b/pkg/neg/syncers/transaction.go @@ -559,9 +559,7 @@ func (s *transactionSyncer) syncNetworkEndpoints(addEndpoints, removeEndpoints m } if operation == attachOp { - // TODO(sawsa307): Pass in subnet to help distinguish which NEGs needs - // update(in default/non-default subnets). - go s.attachNetworkEndpoints(zone, batch) + go s.attachNetworkEndpoints(endpointGroupInfo, batch) } if operation == detachOp { if zone == migrationZone.Zone && subnet == migrationZone.Subnet { @@ -569,9 +567,7 @@ func (s *transactionSyncer) syncNetworkEndpoints(addEndpoints, removeEndpoints m // is already in progress. s.dsMigrator.Pause() } - // TODO(sawsa307): Pass in subnet to help distinguish which NEGs needs - // update(in default/non-default subnets). - go s.detachNetworkEndpoints(zone, batch, zone == migrationZone.Zone && subnet == migrationZone.Subnet) + go s.detachNetworkEndpoints(endpointGroupInfo, batch, zone == migrationZone.Zone && subnet == migrationZone.Subnet) } } return nil @@ -588,18 +584,18 @@ func (s *transactionSyncer) syncNetworkEndpoints(addEndpoints, removeEndpoints m } // attachNetworkEndpoints runs operation for attaching network endpoints. -func (s *transactionSyncer) attachNetworkEndpoints(zone string, networkEndpointMap map[negtypes.NetworkEndpoint]*composite.NetworkEndpoint) { - s.logger.V(2).Info("Attaching endpoints to NEG.", "countOfEndpointsBeingAttached", len(networkEndpointMap), "negSyncerKey", s.NegSyncerKey.String(), "zone", zone) - err := s.operationInternal(attachOp, zone, networkEndpointMap, s.logger) +func (s *transactionSyncer) attachNetworkEndpoints(epGroupInfo negtypes.EndpointGroupInfo, networkEndpointMap map[negtypes.NetworkEndpoint]*composite.NetworkEndpoint) { + s.logger.V(2).Info("Attaching endpoints to NEG.", "countOfEndpointsBeingAttached", len(networkEndpointMap), "negSyncerKey", s.NegSyncerKey.String(), "zone", epGroupInfo.Zone, "subnet", epGroupInfo.Subnet) + err := s.operationInternal(attachOp, epGroupInfo, networkEndpointMap, s.logger) // WARNING: commitTransaction must be called at last for analyzing the operation result s.commitTransaction(err, networkEndpointMap) } // detachNetworkEndpoints runs operation for detaching network endpoints. -func (s *transactionSyncer) detachNetworkEndpoints(zone string, networkEndpointMap map[negtypes.NetworkEndpoint]*composite.NetworkEndpoint, hasMigrationDetachments bool) { - s.logger.V(2).Info("Detaching endpoints from NEG.", "countOfEndpointsBeingDetached", len(networkEndpointMap), "negSyncerKey", s.NegSyncerKey.String(), "zone", zone) - err := s.operationInternal(detachOp, zone, networkEndpointMap, s.logger) +func (s *transactionSyncer) detachNetworkEndpoints(epGroupInfo negtypes.EndpointGroupInfo, networkEndpointMap map[negtypes.NetworkEndpoint]*composite.NetworkEndpoint, hasMigrationDetachments bool) { + s.logger.V(2).Info("Detaching endpoints from NEG.", "countOfEndpointsBeingDetached", len(networkEndpointMap), "negSyncerKey", s.NegSyncerKey.String(), "zone", epGroupInfo.Zone, "subnet", epGroupInfo.Subnet) + err := s.operationInternal(detachOp, epGroupInfo, networkEndpointMap, s.logger) if hasMigrationDetachments { // Unpause the migration since the ongoing migration-detachments have @@ -614,26 +610,38 @@ func (s *transactionSyncer) detachNetworkEndpoints(zone string, networkEndpointM // operationInternal executes NEG API call and commits the transactions // It will record events when operations are completed // If error occurs or any transaction entry requires reconciliation, it will trigger resync -func (s *transactionSyncer) operationInternal(operation transactionOp, zone string, networkEndpointMap map[negtypes.NetworkEndpoint]*composite.NetworkEndpoint, logger klog.Logger) error { +func (s *transactionSyncer) operationInternal(operation transactionOp, epGroupInfo negtypes.EndpointGroupInfo, networkEndpointMap map[negtypes.NetworkEndpoint]*composite.NetworkEndpoint, logger klog.Logger) error { var err error start := time.Now() networkEndpoints := []*composite.NetworkEndpoint{} for _, ne := range networkEndpointMap { networkEndpoints = append(networkEndpoints, ne) } + zone := epGroupInfo.Zone + negName := s.NegSyncerKey.NegName + if flags.F.EnableMultiSubnetClusterPhase1 { + defaultSubnet, err := utils.KeyName(s.networkInfo.SubnetworkURL) + if err != nil { + s.logger.Error(err, "Errored getting default subnet from NetworkInfo when commiting pods") + return err + } + if epGroupInfo.Subnet != defaultSubnet { + negName = s.namer.NonDefaultSubnetNEG(s.NegSyncerKey.Namespace, s.NegSyncerKey.Name, epGroupInfo.Subnet, s.NegSyncerKey.PortTuple.Port) + } + } if operation == attachOp { - err = s.cloud.AttachNetworkEndpoints(s.NegSyncerKey.NegName, zone, networkEndpoints, s.NegSyncerKey.GetAPIVersion(), logger) + err = s.cloud.AttachNetworkEndpoints(negName, zone, networkEndpoints, s.NegSyncerKey.GetAPIVersion(), logger) } if operation == detachOp { - err = s.cloud.DetachNetworkEndpoints(s.NegSyncerKey.NegName, zone, networkEndpoints, s.NegSyncerKey.GetAPIVersion(), logger) + err = s.cloud.DetachNetworkEndpoints(negName, zone, networkEndpoints, s.NegSyncerKey.GetAPIVersion(), logger) } if err == nil { - s.recordEvent(apiv1.EventTypeNormal, operation.String(), fmt.Sprintf("%s %d network endpoint(s) (NEG %q in zone %q)", operation.String(), len(networkEndpointMap), s.NegSyncerKey.NegName, zone)) + s.recordEvent(apiv1.EventTypeNormal, operation.String(), fmt.Sprintf("%s %d network endpoint(s) (NEG %q in zone %q)", operation.String(), len(networkEndpointMap), negName, zone)) s.syncMetricsCollector.UpdateSyncerStatusInMetrics(s.NegSyncerKey, nil, s.inErrorState()) } else { - s.recordEvent(apiv1.EventTypeWarning, operation.String()+"Failed", fmt.Sprintf("Failed to %s %d network endpoint(s) (NEG %q in zone %q): %v", operation.String(), len(networkEndpointMap), s.NegSyncerKey.NegName, zone, err)) + s.recordEvent(apiv1.EventTypeWarning, operation.String()+"Failed", fmt.Sprintf("Failed to %s %d network endpoint(s) (NEG %q in zone %q): %v", operation.String(), len(networkEndpointMap), negName, zone, err)) err := checkEndpointBatchErr(err, operation) syncErr := negtypes.ClassifyError(err) // If the API call fails for invalid endpoint update request in any goroutine, diff --git a/pkg/neg/syncers/transaction_test.go b/pkg/neg/syncers/transaction_test.go index 14970f599..1c4590488 100644 --- a/pkg/neg/syncers/transaction_test.go +++ b/pkg/neg/syncers/transaction_test.go @@ -73,6 +73,7 @@ const ( testUnreadyInstance2 = "unready-instance2" defaultTestSubnet = "default" + additionalTestSubnet = "additional-subnet" secondaryTestSubnet1 = "secondary1" secondaryTestSubnet2 = "secondary2" ) @@ -126,7 +127,7 @@ func TestTransactionSyncNetworkEndpoints(t *testing.T) { t.Errorf("Unexpected neg %q, expected %q", neg.Name, transactionSyncer.NegName) } if neg.NetworkEndpointType != string(testNegType) { - t.Errorf("Unexpected neg type %q, expected %q", neg.Type, testNegType) + t.Errorf("Unexpected neg type %q, expected %q", neg.NetworkEndpointType, testNegType) } if neg.Description == "" { t.Errorf("Neg Description should be populated when NEG CRD is enabled") @@ -257,6 +258,234 @@ func TestTransactionSyncNetworkEndpoints(t *testing.T) { } } +func TestTransactionSyncNetworkEndpointsMSC(t *testing.T) { + vals := gce.DefaultTestClusterValues() + vals.SubnetworkURL = defaultTestSubnetURL + fakeGCE := gce.NewFakeGCECloud(vals) + negtypes.MockNetworkEndpointAPIs(fakeGCE) + fakeCloud := negtypes.NewAdapter(fakeGCE) + testNegTypes := []negtypes.NetworkEndpointType{ + negtypes.VmIpEndpointType, + negtypes.VmIpPortEndpointType, + } + + prevFlag := flags.F.EnableMultiSubnetClusterPhase1 + currNodeTopologyCRName := flags.F.NodeTopologyCRName + defer func() { + flags.F.EnableMultiSubnetClusterPhase1 = prevFlag + flags.F.NodeTopologyCRName = currNodeTopologyCRName + }() + flags.F.EnableMultiSubnetClusterPhase1 = true + flags.F.NodeTopologyCRName = "default" + + nodeTopologyCrWithAdditionalSubnets := nodetopologyv1.NodeTopology{ + TypeMeta: metav1.TypeMeta{ + Kind: "NodeTopology", + APIVersion: "networking.gke.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + }, + Status: nodetopologyv1.NodeTopologyStatus{ + Subnets: []nodetopologyv1.SubnetConfig{ + {Name: defaultTestSubnet, SubnetPath: fmt.Sprintf("projects/mock-project/regions/test-region/subnetworks/%s", defaultTestSubnet)}, + {Name: additionalTestSubnet, SubnetPath: fmt.Sprintf("projects/mock-project/regions/test-region/subnetworks/%s", additionalTestSubnet)}, + }, + }, + } + + for _, testNegType := range testNegTypes { + _, transactionSyncer := newTestTransactionSyncer(fakeCloud, testNegType, false) + if err := zonegetter.AddNodeTopologyCR(transactionSyncer.zoneGetter, &nodeTopologyCrWithAdditionalSubnets); err != nil { + t.Fatalf("Failed to add node topology CR: %v", err) + } + zonegetter.SetNodeTopologyHasSynced(transactionSyncer.zoneGetter, func() bool { return true }) + nonDefaultNegName := transactionSyncer.namer.NonDefaultSubnetNEG(transactionSyncer.NegSyncerKey.Namespace, transactionSyncer.NegSyncerKey.Name, additionalTestSubnet, transactionSyncer.NegSyncerKey.PortTuple.Port) + + if err := transactionSyncer.ensureNetworkEndpointGroups(); err != nil { + t.Errorf("Expect error == nil, but got %v", err) + } + var targetPort string + if testNegType == negtypes.VmIpPortEndpointType { + targetPort = "8080" + } + + // Verify the NEGs are created as expected + ret, _ := transactionSyncer.cloud.AggregatedListNetworkEndpointGroup(transactionSyncer.NegSyncerKey.GetAPIVersion(), klog.TODO()) + // Though the test cases below only add instances in zone1 and zone2, NEGs will be created in zone3 or zone4 as well since fakeZoneGetter includes those zones. + var expectZones []string + if testNegType == negtypes.VmIpEndpointType { + expectZones = []string{negtypes.TestZone1, negtypes.TestZone2, negtypes.TestZone3} + } else { + expectZones = []string{negtypes.TestZone1, negtypes.TestZone2, negtypes.TestZone4} + } + retZones := sets.NewString() + + for key := range ret { + retZones.Insert(key.Zone) + } + for _, zone := range expectZones { + _, ok := retZones[zone] + if !ok { + t.Errorf("Failed to find zone %q from ret %v for negType %v", zone, ret, testNegType) + continue + } + } + for _, neg := range ret { + if neg.NetworkEndpointType != string(testNegType) { + t.Errorf("Unexpected neg type %q for neg %q, expected %q", neg.NetworkEndpointType, neg.Name, testNegType) + } + if neg.Description == "" { + t.Errorf("Neg Description should be populated when NEG CRD is enabled") + } + } + + testCases := []struct { + desc string + addEndpoints map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet + removeEndpoints map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet + expectEndpoints map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet + }{ + { + "empty input", + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, + }, + { + "add some endpoints", + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, targetPort)).Union(generateEndpointSet(net.ParseIP("1.1.2.1"), 10, testInstance2, targetPort)), + {Zone: testZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, targetPort)).Union(generateEndpointSet(net.ParseIP("1.1.4.1"), 10, testInstance4, targetPort)), + }, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, targetPort)).Union(generateEndpointSet(net.ParseIP("1.1.2.1"), 10, testInstance2, targetPort)), + {Zone: testZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, targetPort)).Union(generateEndpointSet(net.ParseIP("1.1.4.1"), 10, testInstance4, targetPort)), + }, + }, + { + "remove some endpoints", + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, targetPort)).Union(generateEndpointSet(net.ParseIP("1.1.2.1"), 10, testInstance2, targetPort)), + }, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, targetPort)).Union(generateEndpointSet(net.ParseIP("1.1.4.1"), 10, testInstance4, targetPort)), + }, + }, + { + "add duplicate endpoints", + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, targetPort)).Union(generateEndpointSet(net.ParseIP("1.1.4.1"), 10, testInstance4, targetPort)), + }, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, targetPort)).Union(generateEndpointSet(net.ParseIP("1.1.4.1"), 10, testInstance4, targetPort)), + }, + }, + { + "add and remove endpoints", + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, targetPort)), + }, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, targetPort)).Union(generateEndpointSet(net.ParseIP("1.1.4.1"), 10, testInstance4, targetPort)), + }, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, targetPort)), + }, + }, + { + "add more endpoints", + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, targetPort)), + }, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, targetPort)), + {Zone: testZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, targetPort)), + }, + }, + { + "add and remove endpoints in both zones", + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.2.1"), 10, testInstance2, targetPort)), + {Zone: testZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.4.1"), 10, testInstance4, targetPort)), + }, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, targetPort)), + {Zone: testZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, targetPort)), + }, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.2.1"), 10, testInstance2, targetPort)), + {Zone: testZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.4.1"), 10, testInstance4, targetPort)), + }, + }, + { + "add endpoints in non-default subnets", + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1, Subnet: additionalTestSubnet}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.2.1.1"), 10, testInstance5, targetPort)), + {Zone: testZone2, Subnet: additionalTestSubnet}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.2.2.1"), 10, testInstance6, targetPort)), + }, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.2.1"), 10, testInstance2, targetPort)), + {Zone: testZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.4.1"), 10, testInstance4, targetPort)), + {Zone: testZone1, Subnet: additionalTestSubnet}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.2.1.1"), 10, testInstance5, targetPort)), + {Zone: testZone2, Subnet: additionalTestSubnet}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.2.2.1"), 10, testInstance6, targetPort)), + }, + }, + } + + for _, tc := range testCases { + // TODO(gauravkghildiyal): When the DualStack Migrator is fully + // implemented, check if we need to cover scenarios where `migrationZone` + // is not empty. + err := transactionSyncer.syncNetworkEndpoints(tc.addEndpoints, tc.removeEndpoints, labels.EndpointPodLabelMap{}, negtypes.EndpointGroupInfo{}) + if err != nil { + t.Errorf("For case %q, syncNetworkEndpoints() got %v, want nil", tc.desc, err) + } + + if err := waitForTransactions(transactionSyncer); err != nil { + t.Errorf("For case %q, waitForTransactions() got %v, want nil", tc.desc, err) + } + + for endpointGroupInfo, endpoints := range tc.expectEndpoints { + negName := transactionSyncer.NegSyncerKey.NegName + if endpointGroupInfo.Subnet != defaultTestSubnet { + negName = nonDefaultNegName + } + list, err := fakeCloud.ListNetworkEndpoints(negName, endpointGroupInfo.Zone, false, transactionSyncer.NegSyncerKey.GetAPIVersion(), klog.TODO()) + if err != nil { + t.Errorf("For case %q, ListNetworkEndpoints() got %v, want nil", tc.desc, err) + } + + endpointSet := negtypes.NewNetworkEndpointSet() + for _, ep := range list { + tmp := negtypes.NetworkEndpoint{IP: ep.NetworkEndpoint.IpAddress, Node: ep.NetworkEndpoint.Instance} + if testNegType == negtypes.VmIpPortEndpointType { + tmp.Port = strconv.FormatInt(ep.NetworkEndpoint.Port, 10) + } + endpointSet.Insert(tmp) + } + + if !endpoints.Equal(endpointSet) { + t.Errorf("For case %q, in zone %q, negType %q, endpointSets endpoints == %v, but got %v, difference: \n(want - got) = %v\n(got - want) = %v", tc.desc, endpointGroupInfo.Zone, testNegType, endpoints, endpointSet, endpoints.Difference(endpointSet), endpointSet.Difference(endpoints)) + } + } + } + transactionSyncer.cloud.DeleteNetworkEndpointGroup(transactionSyncer.NegName, negtypes.TestZone1, transactionSyncer.NegSyncerKey.GetAPIVersion(), klog.TODO()) + transactionSyncer.cloud.DeleteNetworkEndpointGroup(transactionSyncer.NegName, negtypes.TestZone2, transactionSyncer.NegSyncerKey.GetAPIVersion(), klog.TODO()) + transactionSyncer.cloud.DeleteNetworkEndpointGroup(transactionSyncer.NegName, negtypes.TestZone3, transactionSyncer.NegSyncerKey.GetAPIVersion(), klog.TODO()) + transactionSyncer.cloud.DeleteNetworkEndpointGroup(transactionSyncer.NegName, negtypes.TestZone4, transactionSyncer.NegSyncerKey.GetAPIVersion(), klog.TODO()) + transactionSyncer.cloud.DeleteNetworkEndpointGroup(nonDefaultNegName, negtypes.TestZone1, transactionSyncer.NegSyncerKey.GetAPIVersion(), klog.TODO()) + transactionSyncer.cloud.DeleteNetworkEndpointGroup(nonDefaultNegName, negtypes.TestZone2, transactionSyncer.NegSyncerKey.GetAPIVersion(), klog.TODO()) + transactionSyncer.cloud.DeleteNetworkEndpointGroup(nonDefaultNegName, negtypes.TestZone3, transactionSyncer.NegSyncerKey.GetAPIVersion(), klog.TODO()) + transactionSyncer.cloud.DeleteNetworkEndpointGroup(nonDefaultNegName, negtypes.TestZone4, transactionSyncer.NegSyncerKey.GetAPIVersion(), klog.TODO()) + } +} + func TestSyncNetworkEndpointLabel(t *testing.T) { var (