kube-router/pkg/utils/node.go

package utils

import (
	"context"
	"errors"
	"fmt"
	"net"
	"os"

	"github.com/vishvananda/netlink"

	apiv1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	netutils "k8s.io/utils/net"
)

// nodeAddressMap contains Kubernetes node address types (apiv1.NodeAddressType) grouped by Kubernetes Node Object
// address type (internal / external).
type nodeAddressMap map[apiv1.NodeAddressType][]apiv1.NodeAddress

// addressMap contains net.IP addresses grouped by Kubernetes Node Object address type (internal / external).
type addressMap map[apiv1.NodeAddressType][]net.IP

// KRNode is a struct that holds information about a node that is used by kube-router.
type KRNode struct {
	NodeIPv4Addrs addressMap
	NodeIPv6Addrs addressMap
	NodeName      string
	PrimaryIP     net.IP
}

// LocalKRNode is a struct that holds information about this kube-router node.
type LocalKRNode struct {
	KRNode
	NodeInterfaceName string
	linkQ             LocalLinkQuerier
	sloppyTCP         SysctlConfig
}

// NodeIPAware is an interface that provides methods to get the node's IP addresses in various data structures.
type NodeIPAware interface {
	FindBestIPv4NodeAddress() net.IP
	FindBestIPv6NodeAddress() net.IP
	GetNodeIPv4Addrs() []net.IP
	GetNodeIPv6Addrs() []net.IP
	GetNodeIPAddrs() []net.IP
	GetPrimaryNodeIP() net.IP
}

// NodeInterfaceAware is an interface that provides methods to get the node's interface name, MTU, and subnet. This
// interface is a collection of functions that are only available if you are running on the node itself, as kube-router
// determines this by looking at the node's interfaces and parsing the address data there. If you attempt to call these
// functions on a remote node, they will return nil or an error.
type NodeInterfaceAware interface {
	GetNodeInterfaceName() string
	GetNodeMTU() (int, error)
}

// NodeFamilyAware is an interface that provides methods to check if a node is IPv4 or IPv6 capable.
type NodeFamilyAware interface {
	IsIPv4Capable() bool
	IsIPv6Capable() bool
}

// NodeNameAware is an interface that provides a method to get the node's name.
type NodeNameAware interface {
	GetNodeName() string
}

// NodeIPAndFamilyAware is an interface that combines the NodeIPAware and NodeFamilyAware interfaces.
type NodeIPAndFamilyAware interface {
	NodeIPAware
	NodeFamilyAware
}

type NodeConfigAware interface {
	SloppyTCP() *SysctlConfig
}

// NodeAware is an interface that combines the NodeIPAware, NodeInterfaceAware, NodeFamilyAware, and NodeNameAware
// interfaces.
type NodeAware interface {
	NodeConfigAware
	NodeIPAware
	NodeInterfaceAware
	NodeFamilyAware
	NodeNameAware
}

// GetNodeIPv4Addrs returns the node's IPv4 addresses as defined by the Kubernetes Node Object.
func (n *KRNode) GetNodeIPv4Addrs() []net.IP {
	var nodeIPs []net.IP
	nodeIPs = append(nodeIPs, n.NodeIPv4Addrs[apiv1.NodeInternalIP]...)
	nodeIPs = append(nodeIPs, n.NodeIPv4Addrs[apiv1.NodeExternalIP]...)
	return nodeIPs
}

// GetNodeIPv6Addrs returns the node's IPv6 addresses as defined by the Kubernetes Node Object.
func (n *KRNode) GetNodeIPv6Addrs() []net.IP {
	var nodeIPs []net.IP
	nodeIPs = append(nodeIPs, n.NodeIPv6Addrs[apiv1.NodeInternalIP]...)
	nodeIPs = append(nodeIPs, n.NodeIPv6Addrs[apiv1.NodeExternalIP]...)
	return nodeIPs
}

// GetPrimaryNodeIP returns the node's primary IP address which for the purposes of kube-router is defined as the first
// internal address defined on the Kubernetes node object. If no internal address is defined, the first external address
// is used.
func (n *KRNode) GetPrimaryNodeIP() net.IP {
	return n.PrimaryIP
}

// GetNodeInterfaceName returns the node's interface name as defined by the primary IP address. This function is only
// available if you are running on the node itself, as kube-router determines this by looking at the node's interfaces
// and parsing the address data there. If you attempt to call this function on a remote node, it will return nil.
func (n *LocalKRNode) GetNodeInterfaceName() string {
	return n.NodeInterfaceName
}

// IsIPv4Capable returns true if the node has at least one IPv4 address defined in the Kubernetes Node Object.
func (n *KRNode) IsIPv4Capable() bool {
	return len(n.NodeIPv4Addrs[apiv1.NodeInternalIP]) > 0 || len(n.NodeIPv4Addrs[apiv1.NodeExternalIP]) > 0
}

// IsIPv6Capable returns true if the node has at least one IPv6 address defined in the Kubernetes Node Object.
func (n *KRNode) IsIPv6Capable() bool {
	return len(n.NodeIPv6Addrs[apiv1.NodeInternalIP]) > 0 || len(n.NodeIPv6Addrs[apiv1.NodeExternalIP]) > 0
}

// GetNodeName returns the node's name as defined by the Kubernetes Node Object.
func (n *KRNode) GetNodeName() string {
	return n.NodeName
}

func (n *LocalKRNode) SloppyTCP() *SysctlConfig {
	return &n.sloppyTCP
}

// FindBestIPv6NodeAddress returns the best available IPv6 address for the node. If the primary IP is already an IPv6
// address, it will return that. Otherwise, it will return the first internal or external IPv6 address defined in the
// Kubernetes Node Object.
func (n *KRNode) FindBestIPv6NodeAddress() net.IP {
	if n.PrimaryIP != nil && n.PrimaryIP.To4() == nil && n.PrimaryIP.To16() != nil {
		// the NRC's primary IP is already an IPv6 address, so we'll use that
		return n.PrimaryIP
	}
	// the NRC's primary IP is not an IPv6, let's try to find the best available IPv6 address out of our
	// available node addresses to use as the nextHop for our route
	if n.NodeIPv6Addrs != nil {
		if len(n.NodeIPv6Addrs[apiv1.NodeInternalIP]) > 0 {
			return n.NodeIPv6Addrs[apiv1.NodeInternalIP][0]
		} else if len(n.NodeIPv6Addrs[apiv1.NodeExternalIP]) > 0 {
			return n.NodeIPv6Addrs[apiv1.NodeExternalIP][0]
		}
	}
	return nil
}

// FindBestIPv4NodeAddress returns the best available IPv4 address for the node. If the primary IP is already an IPv4
// address, it will return that. Otherwise, it will return the first internal or external IPv4 address defined in the
// Kubernetes Node Object.
func (n *KRNode) FindBestIPv4NodeAddress() net.IP {
	if n.PrimaryIP != nil && n.PrimaryIP.To4() != nil {
		// the NRC's primary IP is already an IPv6 address, so we'll use that
		return n.PrimaryIP
	}
	// the NRC's primary IP is not an IPv6, let's try to find the best available IPv6 address out of our
	// available node addresses to use as the nextHop for our route
	if n.NodeIPv4Addrs != nil {
		if len(n.NodeIPv4Addrs[apiv1.NodeInternalIP]) > 0 {
			return n.NodeIPv4Addrs[apiv1.NodeInternalIP][0]
		} else if len(n.NodeIPv4Addrs[apiv1.NodeExternalIP]) > 0 {
			return n.NodeIPv4Addrs[apiv1.NodeExternalIP][0]
		}
	}
	return nil
}

// GetNodeMTU returns the MTU of the interface that the node's primary IP address is assigned to. This function is only
// available if you are running on the node itself, as kube-router determines this by looking at the node's interfaces
// and parsing the address data there. If you attempt to call this function on a remote node, it will return an error.
func (n *LocalKRNode) GetNodeMTU() (int, error) {
	links, err := n.linkQ.LinkList()
	if err != nil {
		return 0, errors.New("failed to get list of links")
	}
	for _, link := range links {
		addresses, err := n.linkQ.AddrList(link, netlink.FAMILY_ALL)
		if err != nil {
			return 0, errors.New("failed to get list of addr")
		}
		for _, addr := range addresses {
			if addr.IP.Equal(n.PrimaryIP) {
				linkMTU := link.Attrs().MTU
				return linkMTU, nil
			}
		}
	}
	return 0, errors.New("failed to find interface with specified node IP")
}

// GetNodeIPAddrs returns all of the node's IP addresses (whether internal or external) as defined by the Kubernetes
// Node Object.
func (n *KRNode) GetNodeIPAddrs() []net.IP {
	var nodeIPs []net.IP
	ipv4IPs := n.GetNodeIPv4Addrs()
	nodeIPs = append(nodeIPs, ipv4IPs...)
	ipv6IPs := n.GetNodeIPv6Addrs()
	nodeIPs = append(nodeIPs, ipv6IPs...)
	return nodeIPs
}

// NewKRNode creates a new KRNode object from a Kubernetes Node Object. This function is used when kube-router is
// running on the node itself and has access to the node's interfaces and address data. If you attempt to run this on
// a remote node, it will result in an error as it will not be able to find the correct subnet / interface information.
// For this use-case use NewRemoteKRNode instead. It will also return an error if the node does not have any IPv4 or
// IPv6 addresses defined in the Kubernetes Node Object.
func NewKRNode(node *apiv1.Node, linkQ LocalLinkQuerier, enableIPv4, enableIPv6 bool) (*LocalKRNode, error) {
	if linkQ == nil {
		linkQ = &netlink.Handle{}
	}

	primaryNodeIP, err := getPrimaryNodeIP(node)
	if err != nil {
		return nil, fmt.Errorf("error getting primary NodeIP: %w", err)
	}

	ipv4Addrs, ipv6Addrs := getAllNodeIPs(node)
	if enableIPv4 && len(ipv4Addrs[apiv1.NodeInternalIP]) < 1 &&
		len(ipv4Addrs[apiv1.NodeExternalIP]) < 1 {
		return nil, fmt.Errorf("IPv4 was enabled, but no IPv4 address was found on the node")
	}

	if enableIPv6 && len(ipv6Addrs[apiv1.NodeInternalIP]) < 1 &&
		len(ipv6Addrs[apiv1.NodeExternalIP]) < 1 {
		return nil, fmt.Errorf("IPv6 was enabled, but no IPv6 address was found on the node")
	}

	_, nodeInterfaceName, err := GetNodeSubnet(primaryNodeIP, linkQ)
	if err != nil {
		return nil, fmt.Errorf("error getting node subnet: %w", err)
	}

	krNode := &LocalKRNode{
		KRNode: KRNode{
			NodeName:      node.Name,
			PrimaryIP:     primaryNodeIP,
			NodeIPv4Addrs: ipv4Addrs,
			NodeIPv6Addrs: ipv6Addrs,
		},
		linkQ:             linkQ,
		NodeInterfaceName: nodeInterfaceName,
		// Purposefully set the value of sloppyTCP to 0. This ensures the machine's sloppy_tcp setting remains
		// unchanged when there are no services with both Maglev and DSR enabled.
		sloppyTCP: SysctlConfig{
			name:  IPv4IPVSSloppyTCP,
			value: 0,
		},
	}

	return krNode, nil
}

// NewRemoteKRNode creates a new KRNode object from a Kubernetes Node Object. This function is used when kube-router is
// attempting to parse a remote node and does not have access to the node's interfaces and address data. It will return
// an error if the node does not have any IPv4 or IPv6 addresses defined in the Kubernetes Node Object.
func NewRemoteKRNode(node *apiv1.Node) (*KRNode, error) {
	primaryNodeIP, err := getPrimaryNodeIP(node)
	if err != nil {
		return nil, fmt.Errorf("error getting primary NodeIP: %w", err)
	}

	ipv4Addrs, ipv6Addrs := getAllNodeIPs(node)

	krNode := &KRNode{
		NodeName:      node.Name,
		PrimaryIP:     primaryNodeIP,
		NodeIPv4Addrs: ipv4Addrs,
		NodeIPv6Addrs: ipv6Addrs,
	}

	return krNode, nil
}

// GetNodeObject returns the node API object for the node
func GetNodeObject(clientset kubernetes.Interface, hostnameOverride string) (*apiv1.Node, error) {
	// if env NODE_NAME is not set and node is not registered with hostname, then use host name override
	if hostnameOverride != "" {
		node, err := clientset.CoreV1().Nodes().Get(context.Background(), hostnameOverride, metav1.GetOptions{})
		if err != nil {
			return nil, fmt.Errorf("unable to get node %s, due to: %v", hostnameOverride, err)
		}
		return node, nil
	}

	// assuming kube-router is running as pod, first check env NODE_NAME
	nodeName := os.Getenv("NODE_NAME")
	if nodeName != "" {
		node, err := clientset.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{})
		if err != nil {
			return nil, fmt.Errorf("unable to get node %s, due to: %v", nodeName, err)
		}
		return node, nil
	}

	// if env NODE_NAME is not set then check if node is register with hostname
	hostName, _ := os.Hostname()
	node, err := clientset.CoreV1().Nodes().Get(context.Background(), hostName, metav1.GetOptions{})
	if err != nil {
		return nil, fmt.Errorf("failed to identify the node by NODE_NAME, %s or --hostname-override: %v", hostName, err)
	}

	return node, nil
}

// getPrimaryNodeIP returns the most valid external facing IP address for a node.
// Order of preference:
// 1. NodeInternalIP
// 2. NodeExternalIP (usually only set on cloud providers usually)
func getPrimaryNodeIP(node *apiv1.Node) (net.IP, error) {
	addresses := node.Status.Addresses
	addressMap := make(nodeAddressMap)
	for i := range addresses {
		addressMap[addresses[i].Type] = append(addressMap[addresses[i].Type], addresses[i])
	}
	if addresses, ok := addressMap[apiv1.NodeInternalIP]; ok {
		return net.ParseIP(addresses[0].Address), nil
	}
	if addresses, ok := addressMap[apiv1.NodeExternalIP]; ok {
		return net.ParseIP(addresses[0].Address), nil
	}
	return nil, errors.New("host IP unknown")
}

// getAllNodeIPs returns all internal and external IP addresses grouped as IPv4 and IPv6 in a map that is indexed by
// the Kubernetes Node Object address type (internal / external).
func getAllNodeIPs(node *apiv1.Node) (addressMap, addressMap) {
	ipAddrv4 := make(addressMap)
	ipAddrv6 := make(addressMap)
	addresses := node.Status.Addresses
	addressesPerType := make(nodeAddressMap)
	for _, address := range addresses {
		addressesPerType[address.Type] = append(addressesPerType[address.Type], address)
	}
	if internalAddresses, ok := addressesPerType[apiv1.NodeInternalIP]; ok {
		for _, address := range internalAddresses {
			if netutils.IsIPv4String(address.Address) {
				ipAddrv4[apiv1.NodeInternalIP] = append(ipAddrv4[apiv1.NodeInternalIP], net.ParseIP(address.Address))
			}
			if netutils.IsIPv6String(address.Address) {
				ipAddrv6[apiv1.NodeInternalIP] = append(ipAddrv6[apiv1.NodeInternalIP], net.ParseIP(address.Address))
			}
		}
	}
	if externalAddresses, ok := addressesPerType[apiv1.NodeExternalIP]; ok {
		for _, address := range externalAddresses {
			if netutils.IsIPv4String(address.Address) {
				ipAddrv4[apiv1.NodeExternalIP] = append(ipAddrv4[apiv1.NodeExternalIP], net.ParseIP(address.Address))
			}
			if netutils.IsIPv6String(address.Address) {
				ipAddrv6[apiv1.NodeExternalIP] = append(ipAddrv6[apiv1.NodeExternalIP], net.ParseIP(address.Address))
			}
		}
	}

	return ipAddrv4, ipAddrv6
}

// GetNodeSubnet returns the subnet and interface name for a given node IP
func GetNodeSubnet(nodeIP net.IP, linkQ LocalLinkQuerier) (net.IPNet, string, error) {
	if linkQ == nil {
		linkQ = &netlink.Handle{}
	}

	links, err := linkQ.LinkList()
	if err != nil {
		return net.IPNet{}, "", errors.New("failed to get list of links")
	}

	for _, link := range links {
		addresses, err := linkQ.AddrList(link, netlink.FAMILY_ALL)
		if err != nil {
			return net.IPNet{}, "", errors.New("failed to get list of addrs")
		}
		for _, addr := range addresses {
			if addr.IP.Equal(nodeIP) {
				return *addr.IPNet, link.Attrs().Name, nil
			}
		}
	}

	return net.IPNet{}, "", errors.New("failed to find interface with specified node ip")
}