Merge pull request #24 from kubeslice/feature-route-replace

updating ecmp routes if the gateway pod goes down

Dockerfile

@@ -37,7 +37,8 @@ FROM ${PLATFORM}/alpine:3.15
 
 # Add the necessary pakages:
 # tc - is needed for traffic control and shaping on the sidecar.  it is part of the iproute2
-RUN apk add --no-cache ca-certificates
+RUN apk add --no-cache ca-certificates &&\
+    apk add iproute2
 
 # Run the sidecar binary.
 WORKDIR /kubeslice

go.mod

@@ -4,18 +4,18 @@ go 1.17
 
 require (
 	github.com/lorenzosaino/go-sysctl v0.1.1
-	github.com/vishvananda/netlink v1.1.0
+	github.com/vishvananda/netlink v1.2.1-beta.2.0.20220812183158-d44b87fd4d3f
 	go.ligato.io/vpp-agent/v3 v3.2.0
 	go.uber.org/zap v1.16.0
-	golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4
+	golang.org/x/sys v0.0.0-20220804214406-8e32c043e418
 	google.golang.org/grpc v1.46.2
 	google.golang.org/protobuf v1.27.1
 )
 
 require (
 	github.com/BurntSushi/toml v0.3.1 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df // indirect
+	github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect
 	go.uber.org/atomic v1.6.0 // indirect
 	go.uber.org/multierr v1.5.0 // indirect
 	golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect

go.sum

@@ -286,9 +286,13 @@ github.com/unrolled/render v0.0.0-20180914162206-b9786414de4d/go.mod h1:tu82oB5W
 github.com/vishvananda/netlink v0.0.0-20180910184128-56b1bd27a9a3/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
 github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0=
 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
+github.com/vishvananda/netlink v1.2.1-beta.2.0.20220812183158-d44b87fd4d3f h1:kTcjiSlfxwj/o7ezwNrsYjydHYu4Bgu0OkF+q46Vb3k=
+github.com/vishvananda/netlink v1.2.1-beta.2.0.20220812183158-d44b87fd4d3f/go.mod h1:cAAsePK2e15YDAMJNyOpGYEWNe4sIghTY7gpz4cX/Ik=
 github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
+github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3Cym0ZtKyq7L16eZUtYKs+BaHDN6mAns=
+github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/willfaught/gockle v0.0.0-20160623235217-4f254e1e0f0a/go.mod h1:NLcF+3nDpXVIZatjn5Z97gKzFFVU7TzgbAcs8G7/Jrs=
 github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
@@ -385,12 +389,15 @@ golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200610111108-226ff32320da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20220804214406-8e32c043e418 h1:9vYwv7OjYaky/tlAeD7C4oC9EsPTlaFl1H2jS++V+ME=
+golang.org/x/sys v0.0.0-20220804214406-8e32c043e418/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=

pkg/server/get_slice_router_client_conn_info_test.go

@@ -54,6 +54,8 @@ func dialer() func(context.Context, string) (net.Conn, error) {
 
 func TestRouterConnClientInfo(t *testing.T) {
 
+	remoteSubnetRouteMap = make(map[string][]string)
+
 	connList := []*pb.ConnectionInfo{}
 	connInfo := pb.ConnectionInfo{
 		PodName:      "podname",

pkg/server/slicerouter_config.go

@@ -19,6 +19,7 @@ package server
 
 import (
 	"context"
+	"errors"
 	"net"
 	"os"
 	"strconv"
@@ -47,7 +48,7 @@ const (
 
 // remoteSubnetRouteMap holds all the routes that were injected by the vL3 sidecar into the
 // vL3 routing table.
-var remoteSubnetRouteMap map[string]string
+var remoteSubnetRouteMap map[string][]string
 
 // Records the last time the routing table in the slice router was reconciled.
 var lastRoutingTableReconcileTime time.Time
@@ -112,25 +113,70 @@ func vl3DeleteRouteInVpp(dstIP string, nextHopIP string) error {
 	return sendConfigToVppAgent(vppconfig, true)
 }
 
-func vl3InjectRouteInKernel(dstIP string, nextHopIP string) error {
+func vl3InjectRouteInKernel(dstIP string, nextHopIPSlice []*netlink.NexthopInfo) error {
 	_, dstIPNet, err := net.ParseCIDR(dstIP)
 	if err != nil {
 		return err
 	}
-	gwIP := net.ParseIP(nextHopIP)
-
-	route := netlink.Route{Dst: dstIPNet, Gw: gwIP}
 
+	route := netlink.Route{Dst: dstIPNet, MultiPath: nextHopIPSlice}
 	if err := netlink.RouteReplace(&route); err != nil {
-		logger.GlobalLogger.Errorf("Route add failed in kernel. Dst: %v, NextHop: %v, Err: %v", dstIPNet, gwIP, err)
+		logger.GlobalLogger.Errorf("Route add failed in kernel. Dst: %v, NextHop: %v, Err: %v", dstIPNet, nextHopIPSlice, err)
 		return err
 	}
-
-	logger.GlobalLogger.Infof("Route added successfully in the kernel. Dst: %v, NextHop: %v", dstIPNet, gwIP)
+	logger.GlobalLogger.Infof("Route added successfully in the kernel. Dst: %v, NextHop: %v", dstIPNet, nextHopIPSlice)
 
 	return nil
 }
-
+func vl3UpdateEcmpRoute(dstIP string, NsmIPToRemove string) error {
+	_, dstIPNet, err := net.ParseCIDR(dstIP)
+	if err != nil {
+		return err
+	}
+	routes, err := netlink.RouteList(nil, netlink.FAMILY_V4)
+	if err != nil {
+		return err
+	}
+	ecmpRoutes := make([]*netlink.NexthopInfo, 0)
+	for _, route := range routes {
+		if route.Dst.String() == dstIPNet.String() {
+			ecmpRoutes = route.MultiPath
+		}
+	}
+	if len(ecmpRoutes) == 0 {
+		return errors.New("ecmp routes not yet present")
+	}
+	ecmpRoutesCopy := append(make([]*netlink.NexthopInfo, 0), ecmpRoutes...)
+	updatedMultiPath, index := updateMultipath(ecmpRoutes, NsmIPToRemove)
+	err = netlink.RouteReplace(&netlink.Route{Dst: dstIPNet, MultiPath: updatedMultiPath})
+	if err != nil {
+		logger.GlobalLogger.Errorf("Unable to replace ecmp routes, Err: %v", err)
+		return err
+	}
+	remoteSubnetRouteMap[dstIP] = updateIpsInRemoteSubnetMap(dstIPNet, ecmpRoutesCopy[index].Gw)
+	return nil
+}
+func updateMultipath(nextHopIPs []*netlink.NexthopInfo, gwToRemove string) ([]*netlink.NexthopInfo, int) {
+	index := -1
+	for i, _ := range nextHopIPs {
+		if nextHopIPs[i].Gw.String() == gwToRemove {
+			index = i
+			break
+		}
+	}
+	return append(nextHopIPs[:index], nextHopIPs[index+1:]...), index
+}
+func updateIpsInRemoteSubnetMap(dstIPNet *net.IPNet, ipToRemove net.IP) []string {
+	ips, _ := remoteSubnetRouteMap[dstIPNet.String()]
+	index := -1
+	for i, _ := range ips {
+		if ips[i] == ipToRemove.String() {
+			index = i
+			break
+		}
+	}
+	return append(ips[:index], ips[index+1:]...)
+}
 func vl3GetNsmInterfacesInVpp() ([]*sidecar.ConnectionInfo, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
 	defer cancel()
@@ -244,33 +290,54 @@ func vl3ReconcileRoutesInKernel() error {
 		return err
 	}
 
-	routeMap := make(map[string]netlink.Route)
+	routeMap := make(map[string][]netlink.Route)
 	for _, route := range installedRoutes {
 		// Default route will have a Dst of nil so it is
 		// important to have a null check here. Else we will
 		// crash trying to deref a null pointer.
 		if route.Dst == nil {
 			continue
 		}
-		routeMap[route.Dst.String()] = route
+		routeMap[route.Dst.String()] = append(routeMap[route.Dst.String()], route)
 	}
+	logger.GlobalLogger.Infof("installed Route: %v", installedRoutes)
 	logger.GlobalLogger.Infof("Route map: %v", routeMap)
 	logger.GlobalLogger.Infof("Slice Route map: %v", remoteSubnetRouteMap)
 
-	for remoteSubnet, nextHop := range remoteSubnetRouteMap {
-		_, ok := routeMap[remoteSubnet]
-		// If the route is absent or the nexthop is incorrect, reinstall the route.
-		if !ok || routeMap[remoteSubnet].Gw.String() != nextHop {
-			logger.GlobalLogger.Infof("Installed route does not reflect slice state. Reconciling dst: %v, gw: %v", remoteSubnet, nextHop)
-			err := vl3InjectRouteInKernel(remoteSubnet, nextHop)
-			if err != nil {
-				logger.GlobalLogger.Errorf("Failed to install route: dst: %v, gw: %v", remoteSubnet, nextHop)
+	nextHopInfoSlice := []*netlink.NexthopInfo{}
+
+	for remoteSubnet, nextHopList := range remoteSubnetRouteMap {
+		// avoid injecting routes if route map is empty
+		if len(routeMap[remoteSubnet]) == 0 {
+			break
+		}
+		for _, ip := range nextHopList {
+			_, ok := routeMap[remoteSubnet]
+			// If the route is absent or nexthop is incorrect, reinstall the route.
+			if !ok || containsRoute(routeMap[remoteSubnet], ip) {
+				gwObj := &netlink.NexthopInfo{Gw: net.ParseIP(ip)}
+				nextHopInfoSlice = append(nextHopInfoSlice, gwObj)
+
+				logger.GlobalLogger.Infof("Installed route does not reflect slice state. Reconciling dst: %v, gw: %v", remoteSubnet, nextHopInfoSlice)
+				err := vl3InjectRouteInKernel(remoteSubnet, nextHopInfoSlice)
+				if err != nil {
+					logger.GlobalLogger.Errorf("Failed to install route: dst: %v, gw: %v", remoteSubnet, nextHopInfoSlice)
+				}
 			}
 		}
 	}
 	return nil
 }
 
+func getNextHopInfoSlice(nextHopIPList []string) []*netlink.NexthopInfo {
+	nextHopIpSlice := []*netlink.NexthopInfo{}
+	for _, ip := range nextHopIPList {
+		gwObj := &netlink.NexthopInfo{Gw: net.ParseIP(ip)}
+		nextHopIpSlice = append(nextHopIpSlice, gwObj)
+	}
+	return nextHopIpSlice
+}
+
 func sliceRouterReconcileRoutingTable() error {
 	if getSliceRouterDataplaneMode() == SliceRouterDataplaneVpp {
 		return nil
@@ -279,60 +346,85 @@ func sliceRouterReconcileRoutingTable() error {
 	}
 }
 
-// Function to inject remote cluster subnet routes into the local slice router.
-// The next hop IP would be the IP address of the slice-gw that connects to the remote cluster.
-func sliceRouterInjectRoute(remoteSubnet string, nextHopIP string) error {
-	if time.Since(lastRoutingTableReconcileTime).Seconds() < routingTableReconcileInterval {
-		logger.GlobalLogger.Info("Skipping reconcilation, haven't crossed the reconciliation interval yet.")
-		return nil
+func buildNextHopInfo(nextHopIPList []string) []*netlink.NexthopInfo {
+	nextHopIpSlice := []*netlink.NexthopInfo{}
+	for _, nextHop := range nextHopIPList {
+		gwObj := &netlink.NexthopInfo{Gw: net.ParseIP(nextHop)}
+		nextHopIpSlice = append(nextHopIpSlice, gwObj)
 	}
+	return nextHopIpSlice
+}
 
-	err := sliceRouterReconcileRoutingTable()
-	if err != nil {
-		logger.GlobalLogger.Errorf("Failed to reconcile routing table: %v", err)
-	}
+// Function to inject remote cluster subnet routes into the local slice router.
+// The next hop IP would be the IP address of the slice-gw that connects to the remote cluster.
+func sliceRouterInjectRoute(remoteSubnet string, nextHopIPList []string) error {
+	logger.GlobalLogger.Infof("Received NSM IPS from operator: %v", nextHopIPList)
+	if time.Since(lastRoutingTableReconcileTime).Seconds() > routingTableReconcileInterval {
+		err := sliceRouterReconcileRoutingTable()
+		if err != nil {
+			logger.GlobalLogger.Errorf("Failed to reconcile routing table: %v", err)
+		}
 
-	lastRoutingTableReconcileTime = time.Now()
+		lastRoutingTableReconcileTime = time.Now()
 
-	logger.GlobalLogger.Infof("RT reconciled at: %v", lastRoutingTableReconcileTime)
+		logger.GlobalLogger.Infof("RT reconciled at: %v", lastRoutingTableReconcileTime)
+	}
 
 	_, routePresent := remoteSubnetRouteMap[remoteSubnet]
-	if routePresent && remoteSubnetRouteMap[remoteSubnet] == nextHopIP {
-		logger.GlobalLogger.Infof("Ignoring route add request. Route already installed. RemoteSubnet: %v, NextHop: %v",
-			remoteSubnet, nextHopIP)
-		return nil
-	}
+	nextHopInfoSlice := getNextHopInfoSlice(nextHopIPList)
 
-	if getSliceRouterDataplaneMode() == SliceRouterDataplaneVpp {
-		// If a route was previously installed for the remote subnet then we should
-		// delete it before adding a route with a new nexthop IP.
-		// VPP treats a route modify as a route add operation, creating multiple
-		// entries for a destination prefix and treating them as equal cost multipath
-		// routes.
-		// In our case, we should have only one route with the nexthop as the nsm IP on
-		// the slice gw pod connecting the remote subnet.
-		if remoteSubnetRouteMap[remoteSubnet] != "" {
-			err := vl3DeleteRouteInVpp(remoteSubnet, remoteSubnetRouteMap[remoteSubnet])
+	for i := 0; i < len(nextHopIPList); i++ {
+
+		if routePresent && checkRouteAdd(remoteSubnetRouteMap[remoteSubnet], nextHopIPList[i]) {
+			logger.GlobalLogger.Infof("Ignoring route add request. Route already installed. RemoteSubnet: %v, NextHop: %v",
+				remoteSubnet, nextHopIPList[i])
+			continue
+		}
+		if getSliceRouterDataplaneMode() == SliceRouterDataplaneVpp {
+			// If a route was previously installed for the remote subnet then we should
+			// delete it before adding a route with a new nexthop IP.
+			// VPP treats a route modify as a route add operation, creating multiple
+			// entries for a destination prefix and treating them as equal cost multipath
+			// routes.
+			// In our case, we should have only one route with the nexthop as the nsm IP on
+			// the slice gw pod connecting the remote subnet.
+			if len(remoteSubnetRouteMap[remoteSubnet]) != 0 {
+				err := vl3DeleteRouteInVpp(remoteSubnet, remoteSubnetRouteMap[remoteSubnet][i])
+				if err != nil {
+					logger.GlobalLogger.Errorf("Failed to delete route with old gw IP. RemoteSubent: %v, NextHop: %v",
+						remoteSubnet, remoteSubnetRouteMap[remoteSubnet][i])
+				}
+			}
+			err := vl3InjectRouteInVpp(remoteSubnet, nextHopIPList[i])
 			if err != nil {
-				logger.GlobalLogger.Errorf("Failed to delete route with old gw IP. RemoteSubent: %v, NextHop: %v",
-					remoteSubnet, remoteSubnetRouteMap[remoteSubnet])
-				return err
+				logger.GlobalLogger.Errorf("Failed to inject route in vpp: %v", err)
+			}
+		} else {
+			err := vl3InjectRouteInKernel(remoteSubnet, nextHopInfoSlice)
+			if err != nil {
+				logger.GlobalLogger.Errorf("Failed to inject route in kernel: %v", err)
 			}
 		}
-		err := vl3InjectRouteInVpp(remoteSubnet, nextHopIP)
-		if err != nil {
-			return err
-		}
-	} else {
-		err := vl3InjectRouteInKernel(remoteSubnet, nextHopIP)
-		if err != nil {
-			return err
+		remoteSubnetRouteMap[remoteSubnet] = append(remoteSubnetRouteMap[remoteSubnet], nextHopIPList[i])
+	}
+	return nil
+}
+func checkRouteAdd(nextHopIpList []string, s string) bool {
+	for _, nextHop := range nextHopIpList {
+		if nextHop == s {
+			return true
 		}
 	}
+	return false
+}
 
-	remoteSubnetRouteMap[remoteSubnet] = nextHopIP
-
-	return nil
+func containsRoute(nextHopIpList []netlink.Route, s string) bool {
+	for _, nextHop := range nextHopIpList {
+		if nextHop.Gw.String() == s {
+			return true
+		}
+	}
+	return false
 }
 
 func sliceRouterGetClientConnections() ([]*sidecar.ConnectionInfo, error) {
@@ -356,8 +448,13 @@ func BootstrapSliceRouterPod() error {
 			logger.GlobalLogger.Fatalf("Failed to enable IP forwarding in the kernel", err)
 			return err
 		}
+		err = sysctl.Set("net.ipv4.fib_multipath_hash_policy", "1")
+		if err != nil {
+			logger.GlobalLogger.Fatalf("failed to set hash policy to L4 for mutipath routes", err)
+			return err
+		}
 	}
-	remoteSubnetRouteMap = make(map[string]string)
+	remoteSubnetRouteMap = make(map[string][]string)
 	lastRoutingTableReconcileTime = time.Now()
 	return nil
 }