feat(linux_tunnels): convert to use netlink library

pkg/tunnels/linux_tunnels.go

@@ -1,7 +1,11 @@
+// Package tunnels provides functionality for setting up and managing overlay tunnels in Linux.
+// It includes support for both IPIP and FOU (Foo over Ethernet) encapsulation types.
+//
+// As much functionality as possible is done via the netlink library, however, FOU tunnels require using the iproute2
+// user space tooling since they are not currently supported by the netlink library.
 package tunnels
 
 import (
-	"bufio"
 	"crypto/sha256"
 	"fmt"
 	"net"
@@ -31,6 +35,14 @@ const (
 	// The maximum and minimum port numbers for encap ports
 	maxPort = uint16(65535)
 	minPort = uint16(1024)
+
+	// Unix tunnel encap types, unfortunately, these are not understood by the netlink library, so we need to use
+	// our own enums which as far as I can tell come from here:
+	// https://github.com/iproute2/iproute2/blob/e6a170a9d4e75d206631da77e469813279c12134/include/uapi/linux/if_tunnel.h#L84-L89
+	UnixTunnelEncapTypeNone uint16 = 0
+	UnixTunnelEncapTypeFOU  uint16 = 1
+	UnixTunnelEncapTypeGUE  uint16 = 2
+	UnixTunnelEncapTypeMPLS uint16 = 3
 )
 
 var (
@@ -106,7 +118,6 @@ func (o *OverlayTunnel) EncapPort() EncapPort {
 // setupOverlayTunnel attempts to create a tunnel link and corresponding routes for IPIP based overlay networks
 func (o *OverlayTunnel) SetupOverlayTunnel(tunnelName string, nextHop net.IP,
 	nextHopSubnet *net.IPNet) (netlink.Link, error) {
-	var out []byte
 	link, err := netlink.LinkByName(tunnelName)
 
 	var bestIPForFamily net.IP
@@ -142,8 +153,14 @@ func (o *OverlayTunnel) SetupOverlayTunnel(tunnelName string, nextHop net.IP,
 
 		switch o.encapType {
 		case EncapTypeIPIP:
-			if linkFOUEnabled(tunnelName) {
+			if fouEnabled, err := linkFOUEnabled(tunnelName); err != nil || fouEnabled {
-				klog.Infof("Was configured to use ipip tunnels, but found existing fou tunnels in place, cleaning up")
+				if err != nil {
+					klog.Errorf("failed to check if fou is enabled on the link %s: %v, going to try to clean up and "+
+						"recreate the tunnel", tunnelName, err)
+				} else {
+					klog.Infof("Was configured to use ipip tunnels, but found existing fou tunnels in place, " +
+						"cleaning up")
+				}
 				recreate = true
 
 				// Even though we are setup for IPIP tunels we have existing tunnels that are FoU tunnels, remove them
@@ -163,58 +180,97 @@ func (o *OverlayTunnel) SetupOverlayTunnel(tunnelName string, nextHop net.IP,
 				}
 			}
 		case EncapTypeFOU:
-			if !linkFOUEnabled(tunnelName) {
+			if fouEnabled, err := linkFOUEnabled(tunnelName); err != nil || !fouEnabled {
-				klog.Infof("Was configured to use fou tunnels, but found existing ipip tunnels in place, cleaning up")
+				if err != nil {
+					klog.Errorf("failed to check if fou is enabled on the link %s: %v, going to try to clean up and "+
+						"recreate the tunnel", tunnelName, err)
+				} else {
+					klog.Infof("Was configured to use fou tunnels, but found existing ipip tunnels in place, " +
+						"cleaning up")
+				}
 				recreate = true
 				// Even though we are setup for FoU tunels we have existing tunnels that are IPIP tunnels, remove them
 				// so that we can recreate them as IPIP
 				CleanupTunnel(nextHopSubnet, tunnelName)
 			}
+		default:
+			return nil, fmt.Errorf("unknown tunnel encapsulation was passed: %s, unable to continue with overlay "+
+				"setup", o.encapType)
 		}
 	}
 
 	// an error here indicates that the tunnel didn't exist, so we need to create it, if it already exists there's
 	// nothing to do here
 	if err != nil || recreate {
-		klog.Infof("Creating tunnel %s of type %s with encap %s for destination %s",
+		klog.Infof("Creating tunnel %s with encap %s for destination %s",
-			tunnelName, fouLinkType, o.encapType, nextHop.String())
+			tunnelName, o.encapType, nextHop.String())
-		cmdArgs := ipBase
+
 		switch o.encapType {
 		case EncapTypeIPIP:
-			// Plain IPIP tunnel without any encapsulation
+			// Create plain IPIP tunnel using netlink
-			cmdArgs = append(cmdArgs, "tunnel", "add", tunnelName, "mode", ipipMode, "local", bestIPForFamily.String(),
+			var tunnelLink netlink.Link
-				"remote", nextHop.String())
+			if isIPv6 {
+				tunnelLink = &netlink.Ip6tnl{
+					LinkAttrs: netlink.LinkAttrs{Name: tunnelName},
+					Local:     bestIPForFamily,
+					Remote:    nextHop,
+				}
+			} else {
+				tunnelLink = &netlink.Iptun{
+					LinkAttrs: netlink.LinkAttrs{Name: tunnelName},
+					Local:     bestIPForFamily,
+					Remote:    nextHop,
+				}
+			}
+
+			if err := netlink.LinkAdd(tunnelLink); err != nil {
+				return nil, fmt.Errorf("route not injected for the route advertised by the node %s "+
+					"Failed to create tunnel interface %s. error: %v", nextHop, tunnelName, err)
+			}
 
 		case EncapTypeFOU:
 			// Ensure that the FOU tunnel port is set correctly
 			if !fouPortAndProtoExist(o.encapPort, isIPv6) {
-				fouArgs := ipBase
+				// Create FOU port using netlink
-				fouArgs = append(fouArgs, "fou", "add", "port", strFormattedEncapPort, "gue")
+				var family int
-				out, err := exec.Command("ip", fouArgs...).CombinedOutput()
+				if isIPv6 {
-				if err != nil {
+					family = netlink.FAMILY_V6
+				} else {
+					family = netlink.FAMILY_V4
+				}
+
+				fouPort := &netlink.Fou{
+					Family:    family,
+					Port:      int(o.encapPort),
+					EncapType: netlink.FOU_ENCAP_GUE,
+				}
+
+				if err := netlink.FouAdd(*fouPort); err != nil {
 					return nil, fmt.Errorf("route not injected for the route advertised by the node %s "+
-						"Failed to set FoU tunnel port - error: %s, output: %s", tunnelName, err, string(out))
+						"Failed to set FoU tunnel port - error: %v", nextHop, err)
 				}
 			}
 
-			// Prep IPIP tunnel for FOU encapsulation
+			// For FOU tunnels, we still need to use exec.Command because the netlink library doesn't support ipip &
+			// ip6ip6 secondary encapsulation modes on links. It does support GUE, but until it supports secondary
+			// encapsulation modes, we need to use the iproute2 tooling to create the tunnel.
+			cmdArgs := ipBase
 			cmdArgs = append(cmdArgs, "link", "add", "name", tunnelName, "type", fouLinkType, "remote", nextHop.String(),
 				"local", bestIPForFamily.String(), "ttl", "225", "encap", "gue", "encap-sport", "auto", "encap-dport",
 				strFormattedEncapPort, "mode", ipipMode)
 
+			klog.V(2).Infof("Executing the following command to create tunnel: ip %s", cmdArgs)
+			out, err := exec.Command("ip", cmdArgs...).CombinedOutput()
+			if err != nil {
+				return nil, fmt.Errorf("route not injected for the route advertised by the node %s "+
+					"Failed to create tunnel interface %s. error: %s, output: %s",
+					nextHop, tunnelName, err, string(out))
+			}
 		default:
 			return nil, fmt.Errorf("unknown tunnel encapsulation was passed: %s, unable to continue with overlay "+
 				"setup", o.encapType)
 		}
 
-		klog.V(2).Infof("Executing the following command to create tunnel: ip %s", cmdArgs)
-		out, err := exec.Command("ip", cmdArgs...).CombinedOutput()
-		if err != nil {
-			return nil, fmt.Errorf("route not injected for the route advertised by the node %s "+
-				"Failed to create tunnel interface %s. error: %s, output: %s",
-				nextHop, tunnelName, err, string(out))
-		}
-
 		link, err = netlink.LinkByName(tunnelName)
 		if err != nil {
 			return nil, fmt.Errorf("route not injected for the route advertised by the node %s "+
@@ -227,19 +283,33 @@ func (o *OverlayTunnel) SetupOverlayTunnel(tunnelName string, nextHop net.IP,
 
 	// Now that the tunnel link exists, we need to add a route to it, so the node knows where to send traffic bound for
 	// this interface
-	//nolint:gocritic // we understand that we are appending to a new slice
+	var routeFamily int
-	cmdArgs := append(ipBase, "route", "list", "table", strconv.Itoa(routes.CustomTableID))
+	if isIPv6 {
-	out, err = exec.Command("ip", cmdArgs...).CombinedOutput()
+		routeFamily = netlink.FAMILY_V6
-	// This used to be "dev "+tunnelName+" scope" but this isn't consistent with IPv6's output, so we changed it to just
+	} else {
-	// "dev "+tunnelName, but at this point I'm unsure if there was a good reason for adding scope on before, so that's
+		routeFamily = netlink.FAMILY_V4
-	// why this comment is here.
+	}
-	if err != nil || !strings.Contains(string(out), "dev "+tunnelName) {
+
-		//nolint:gocritic // we understand that we are appending to a new slice
+	// Check if route already exists in the custom table
-		cmdArgs = append(ipBase, "route", "add", nextHop.String(), "dev", tunnelName, "table",
+	route := &netlink.Route{
-			strconv.Itoa(routes.CustomTableID))
+		Family:    routeFamily,
-		if out, err = exec.Command("ip", cmdArgs...).CombinedOutput(); err != nil {
+		LinkIndex: link.Attrs().Index,
-			return nil, fmt.Errorf("failed to add route in custom route table, err: %s, output: %s", err, string(out))
+		Table:     routes.CustomTableID,
+		Dst:       utils.GetSingleIPNet(nextHop),
+	}
+	routeList, err := netlink.RouteListFiltered(routeFamily, route,
+		netlink.RT_FILTER_OIF|netlink.RT_FILTER_TABLE|netlink.RT_FILTER_DST)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list routes in custom table: %v", err)
+	}
+
+	if len(routeList) < 1 {
+		// Add route to the custom table
+		if err = netlink.RouteAdd(route); err != nil {
+			return nil, fmt.Errorf("failed to add route in custom route table, err: %v", err)
 		}
+	} else {
+		klog.V(2).Infof("Route for %s already exists in custom table", nextHop.String())
 	}
 
 	return link, nil
@@ -284,48 +354,25 @@ func GenerateTunnelName(nodeIP string) string {
 
 // fouPortAndProtoExist checks to see if the given FoU port is already configured on the system via iproute2
 // tooling for the given protocol
-//
-// fou show, shows both IPv4 and IPv6 ports in the same show command, they look like:
-// port 5556 gue
-// port 5556 gue -6
-// where the only thing that distinguishes them is the -6 or not on the end
-// WARNING we're parsing a CLI tool here not an API, this may break at some point in the future
 func fouPortAndProtoExist(port EncapPort, isIPv6 bool) bool {
 	const ipRoute2IPv6Prefix = "-6"
 	strPort := strconv.FormatInt(int64(port), 10)
-	fouArgs := make([]string, 0)
 	klog.V(2).Infof("Checking FOU Port and Proto... %s - %t", strPort, isIPv6)
 
+	nFamily := netlink.FAMILY_V4
 	if isIPv6 {
-		fouArgs = append(fouArgs, ipRoute2IPv6Prefix)
+		nFamily = netlink.FAMILY_V6
 	}
-	fouArgs = append(fouArgs, "fou", "show")
 
-	out, err := exec.Command("ip", fouArgs...).CombinedOutput()
+	fList, err := netlink.FouList(nFamily)
-	// iproute2 returns an error if no fou configuration exists
 	if err != nil {
+		klog.Errorf("failed to list fou ports: %v", err)
 		return false
 	}
 
-	strOut := string(out)
+	for _, fou := range fList {
-	klog.V(2).Infof("Combined output of ip fou show: %s", strOut)
+		klog.V(2).Infof("Found fou port: %s", fou)
-	scanner := bufio.NewScanner(strings.NewReader(strOut))
+		if fou.Port == int(port) && fou.Family == nFamily {
-
-	// loop over all lines of output
-	for scanner.Scan() {
-		scannedLine := scanner.Text()
-		// if the output doesn't contain our port at all, then continue
-		if !strings.Contains(scannedLine, strPort) {
-			continue
-		}
-
-		// if this is IPv6 port and it has the correct IPv6 suffix (see example above) then return true
-		if isIPv6 && strings.HasSuffix(scannedLine, ipRoute2IPv6Prefix) {
-			return true
-		}
-
-		// if this is not IPv6 and it does not have an IPv6 suffix (see example above) then return true
-		if !isIPv6 && !strings.HasSuffix(scannedLine, ipRoute2IPv6Prefix) {
 			return true
 		}
 	}
@@ -336,26 +383,29 @@ func fouPortAndProtoExist(port EncapPort, isIPv6 bool) bool {
 // linkFOUEnabled checks to see whether the given link has FoU (Foo over Ethernet) enabled on it, specifically since
 // kube-router only works with GUE (Generic UDP Encapsulation) we look for that and not just FoU in general. If the
 // linkName is enabled with FoU GUE then we return true, otherwise false
-//
+func linkFOUEnabled(linkName string) (bool, error) {
-// Output for a FoU Enabled GUE tunnel looks like:
+	const gueEncapType = "gue"
-// ipip ipip remote <ip> local <ip> dev <dev> ttl 225 pmtudisc encap gue encap-sport auto encap-dport 5555 ...
+	link, err := netlink.LinkByName(linkName)
-// Output for a normal IPIP tunnel looks like:
-// ipip ipip remote <ip> local <ip> dev <dev> ttl inherit ...
-func linkFOUEnabled(linkName string) bool {
-	const fouEncapEnabled = "encap gue"
-	cmdArgs := []string{"-details", "link", "show", linkName}
-
-	out, err := exec.Command("ip", cmdArgs...).CombinedOutput()
-
 	if err != nil {
-		klog.Warningf("recevied an error while trying to look at the link details of %s, this shouldn't have happened",
+		return false, fmt.Errorf("failed to get link by name: %v", err)
-			linkName)
-		return false
 	}
 
-	if strings.Contains(string(out), fouEncapEnabled) {
+	switch link := link.(type) {
-		return true
+	case *netlink.Iptun:
+		klog.V(2).Infof("Link %s is an IPTun with encap type: %d and encap dport: %d",
+			linkName, link.EncapType, link.EncapDport)
+		if link.EncapType == UnixTunnelEncapTypeGUE {
+			return true, nil
+		}
+	case *netlink.Ip6tnl:
+		klog.V(2).Infof("Link %s is an IP6Tun with encap type: %d and encap dport: %d",
+			linkName, link.EncapType, link.EncapDport)
+		if link.EncapType == UnixTunnelEncapTypeGUE {
+			return true, nil
+		}
+	default:
+		return false, fmt.Errorf("Link %s is not an IPTun or IP6Tun, this is not expected", linkName)
 	}
 
-	return false
+	return false, nil
 }