kube-router/pkg/controllers/routing/bgp_policies.go

package routing

import (
	"context"
	"errors"
	"fmt"
	"net"
	"reflect"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"github.com/ccoveille/go-safecast"
	gobgpapi "github.com/osrg/gobgp/v3/api"
	v1core "k8s.io/api/core/v1"
	"k8s.io/klog/v2"

	"github.com/cloudnativelabs/kube-router/v2/pkg/utils"
)

const (
	podCIDRSet   = "podcidrdefinedset"
	podCIDRSetV6 = "podcidrdefinedsetv6"

	serviceVIPsSet   = "servicevipsdefinedset"
	serviceVIPsSetV6 = "servicevipsdefinedsetv6"

	allPeerSet        = "allpeerset"
	allPeerSetV6      = "allpeersetv6"
	externalPeerSet   = "externalpeerset"
	externalPeerSetV6 = "externalpeersetv6"
	iBGPPeerSet       = "iBGPpeerset"
	iBGPPeerSetV6     = "iBGPpeersetv6"

	customImportRejectSet = "customimportrejectdefinedset"
	defaultRouteSet       = "defaultroutedefinedset"
	defaultRouteSetV6     = "defaultroutedefinedsetv6"

	kubeRouterExportPolicy = "kube_router_export"
	kubeRouterImportPolicy = "kube_router_import"

	maxIPv4MaskSize = uint32(32)
	maxIPv6MaskSize = uint32(128)
)

var (
	rePolicyVersionExtractor = regexp.MustCompile(`(kube_router_(?:import|export))(\d*)`)
)

// AddPolicies adds BGP import and export policies
func (nrc *NetworkRoutingController) AddPolicies() error {
	// we are rr server do not add export policies
	if nrc.bgpRRServer {
		return nil
	}

	err := nrc.addPodCidrDefinedSet()
	if err != nil {
		klog.Errorf("Failed to add `podcidrdefinedset` defined set: %s", err)
	}

	err = nrc.addServiceVIPsDefinedSet()
	if err != nil {
		klog.Errorf("Failed to add `servicevipsdefinedset` defined set: %s", err)
	}

	err = nrc.addDefaultRouteDefinedSet()
	if err != nil {
		klog.Errorf("Failed to add `defaultroutedefinedset` defined set: %s", err)
	}

	err = nrc.addCustomImportRejectDefinedSet()
	if err != nil {
		klog.Errorf("Failed to add `customimportrejectdefinedset` defined set: %s", err)
	}

	iBGPPeerCIDRs, err := nrc.addiBGPPeersDefinedSet()
	if err != nil {
		klog.Errorf("Failed to add `iBGPpeerset` defined set: %s", err)
	}

	externalBGPPeerCIDRs, err := nrc.addExternalBGPPeersDefinedSet()
	if err != nil {
		klog.Errorf("Failed to add `externalpeerset` defined set: %s", err)
	}

	err = nrc.addAllBGPPeersDefinedSet(iBGPPeerCIDRs, externalBGPPeerCIDRs)
	if err != nil {
		klog.Errorf("Failed to add `allpeerset` defined set: %s", err)
	}

	err = nrc.addExportPolicies()
	if err != nil {
		return err
	}

	err = nrc.addImportPolicies()
	if err != nil {
		return err
	}

	return nil
}

// create a defined set to represent just the pod CIDR associated with the node
func (nrc *NetworkRoutingController) addPodCidrDefinedSet() error {
	var currentDefinedSet *gobgpapi.DefinedSet

	for setName, cidrs := range map[string][]string{
		podCIDRSet:   nrc.podIPv4CIDRs,
		podCIDRSetV6: nrc.podIPv6CIDRs,
	} {
		err := nrc.bgpServer.ListDefinedSet(context.Background(),
			&gobgpapi.ListDefinedSetRequest{DefinedType: gobgpapi.DefinedType_PREFIX, Name: setName},
			func(ds *gobgpapi.DefinedSet) {
				currentDefinedSet = ds
			})
		if err != nil {
			return err
		}
		if currentDefinedSet == nil {
			var prefixes []*gobgpapi.Prefix
			for _, cidr := range cidrs {
				_, ipNet, err := net.ParseCIDR(cidr)
				if err != nil {
					return fmt.Errorf("couldn't parse CIDR: %s - %v", cidr, err)
				}
				cidrLen, _ := ipNet.Mask.Size()
				var cidrMax int
				if setName == podCIDRSet {
					cidrMax = 32
				} else {
					cidrMax = 128
				}
				if cidrLen < 0 || cidrLen > cidrMax {
					return fmt.Errorf("the pod CIDR IP given is not a proper mask: %d", cidrLen)
				}
				uCIDRLen, err := safecast.ToUint32(cidrLen)
				if err != nil {
					return fmt.Errorf("failed to convert CIDR length to uint32: %v", err)
				}
				prefixes = append(prefixes, &gobgpapi.Prefix{
					IpPrefix:      cidr,
					MaskLengthMin: uCIDRLen,
					MaskLengthMax: uCIDRLen,
				})
			}
			podCidrDefinedSet := &gobgpapi.DefinedSet{
				DefinedType: gobgpapi.DefinedType_PREFIX,
				Name:        setName,
				Prefixes:    prefixes,
			}
			err = nrc.bgpServer.AddDefinedSet(context.Background(),
				&gobgpapi.AddDefinedSetRequest{DefinedSet: podCidrDefinedSet})
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// create a defined set to represent all the advertisable IP associated with the services
func (nrc *NetworkRoutingController) addServiceVIPsDefinedSet() error {
	for setName, cidrMask := range map[string]uint32{
		serviceVIPsSet:   maxIPv4MaskSize,
		serviceVIPsSetV6: maxIPv6MaskSize,
	} {
		currentDefinedSet, err := nrc.getDefinedSetFromGoBGP(setName, gobgpapi.DefinedType_PREFIX)
		if err != nil {
			return err
		}

		advIPPrefixList := make([]*gobgpapi.Prefix, 0)
		advIps, _, _ := nrc.getVIPs()
		for _, ipStr := range advIps {
			ip := net.ParseIP(ipStr)
			if ip == nil {
				// nothing to do for invalid IPs
				klog.Warningf("found an invalid IP address in list of service VIPs returned from k8s: %s skipping",
					ipStr)
				continue
			}
			switch {
			// Only add IPv4 VIPs when we are populating the serviceVIPsSet set
			case ip.To4() != nil && setName == serviceVIPsSet:
				advIPPrefixList = append(advIPPrefixList,
					&gobgpapi.Prefix{
						IpPrefix:      fmt.Sprintf("%s/%d", ip, cidrMask),
						MaskLengthMin: cidrMask,
						MaskLengthMax: cidrMask,
					})
			// Only add IPv6 VIPs when we are populating the serviceVIPsSetV6 set
			case ip.To4() == nil && ip.To16() != nil && setName == serviceVIPsSetV6:
				advIPPrefixList = append(advIPPrefixList,
					&gobgpapi.Prefix{
						IpPrefix:      fmt.Sprintf("%s/%d", ip, cidrMask),
						MaskLengthMin: cidrMask,
						MaskLengthMax: cidrMask,
					})
			}
		}
		if currentDefinedSet == nil {
			clusterIPPrefixSet := &gobgpapi.DefinedSet{
				DefinedType: gobgpapi.DefinedType_PREFIX,
				Name:        setName,
				Prefixes:    advIPPrefixList,
			}
			err = nrc.bgpServer.AddDefinedSet(context.Background(),
				&gobgpapi.AddDefinedSetRequest{DefinedSet: clusterIPPrefixSet})
			if err != nil {
				return err
			}
			continue
		}

		currentPrefixes := currentDefinedSet.Prefixes
		sort.SliceStable(advIPPrefixList, func(i, j int) bool {
			return advIPPrefixList[i].IpPrefix < advIPPrefixList[j].IpPrefix
		})
		sort.SliceStable(currentPrefixes, func(i, j int) bool {
			return currentPrefixes[i].IpPrefix < currentPrefixes[j].IpPrefix
		})
		if reflect.DeepEqual(advIPPrefixList, currentPrefixes) {
			continue
		}
		toAdd := make([]*gobgpapi.Prefix, 0)
		toDelete := make([]*gobgpapi.Prefix, 0)
		for _, prefix := range advIPPrefixList {
			add := true
			for _, currentPrefix := range currentDefinedSet.Prefixes {
				if currentPrefix.IpPrefix == prefix.IpPrefix {
					add = false
				}
			}
			if add {
				toAdd = append(toAdd, prefix)
			}
		}
		for _, currentPrefix := range currentDefinedSet.Prefixes {
			shouldDelete := true
			for _, prefix := range advIPPrefixList {
				if currentPrefix.IpPrefix == prefix.IpPrefix {
					shouldDelete = false
				}
			}
			if shouldDelete {
				toDelete = append(toDelete, currentPrefix)
			}
		}
		serviceVIPIPPrefixSet := &gobgpapi.DefinedSet{
			DefinedType: gobgpapi.DefinedType_PREFIX,
			Name:        setName,
			Prefixes:    toAdd,
		}
		err = nrc.bgpServer.AddDefinedSet(context.Background(),
			&gobgpapi.AddDefinedSetRequest{DefinedSet: serviceVIPIPPrefixSet})
		if err != nil {
			return err
		}
		serviceVIPIPPrefixSet = &gobgpapi.DefinedSet{
			DefinedType: gobgpapi.DefinedType_PREFIX,
			Name:        setName,
			Prefixes:    toDelete,
		}
		err = nrc.bgpServer.DeleteDefinedSet(context.Background(),
			&gobgpapi.DeleteDefinedSetRequest{DefinedSet: serviceVIPIPPrefixSet, All: false})
		if err != nil {
			return err
		}
	}

	return nil
}

// create a defined set to represent just the host default route
func (nrc *NetworkRoutingController) addDefaultRouteDefinedSet() error {
	for setName, defaultRoute := range map[string]string{
		defaultRouteSet:   utils.IPv4DefaultRoute,
		defaultRouteSetV6: utils.IPv6DefaultRoute,
	} {
		currentDefinedSet, err := nrc.getDefinedSetFromGoBGP(setName, gobgpapi.DefinedType_PREFIX)
		if err != nil {
			return err
		}

		if currentDefinedSet == nil {
			cidrLen := 0
			defaultRouteDefinedSet := &gobgpapi.DefinedSet{
				DefinedType: gobgpapi.DefinedType_PREFIX,
				Name:        setName,
				Prefixes: []*gobgpapi.Prefix{
					{
						IpPrefix:      defaultRoute,
						MaskLengthMin: uint32(cidrLen),
						MaskLengthMax: uint32(cidrLen),
					},
				},
			}
			err = nrc.bgpServer.AddDefinedSet(context.Background(),
				&gobgpapi.AddDefinedSetRequest{DefinedSet: defaultRouteDefinedSet})
			if err != nil {
				return err
			}
		}
	}
	return nil
}

// create a defined set to represent custom annotated routes to be rejected on import
func (nrc *NetworkRoutingController) addCustomImportRejectDefinedSet() error {
	var currentDefinedSet *gobgpapi.DefinedSet
	currentDefinedSet, err := nrc.getDefinedSetFromGoBGP(customImportRejectSet, gobgpapi.DefinedType_PREFIX)
	if err != nil {
		return err
	}

	if currentDefinedSet == nil {
		prefixes := make([]*gobgpapi.Prefix, 0)
		for _, ipNet := range nrc.nodeCustomImportRejectIPNets {
			prefix := new(gobgpapi.Prefix)
			prefix.IpPrefix = ipNet.String()
			mask, _ := ipNet.Mask.Size()

			uIntMask, err := safecast.ToUint32(mask)
			if err != nil {
				return fmt.Errorf("failed to convert mask to uint32: %v", err)
			}

			prefix.MaskLengthMin = uIntMask
			prefix.MaskLengthMax = uint32(ipv4MaskMinBits)
			prefixes = append(prefixes, prefix)
		}
		customImportRejectDefinedSet := &gobgpapi.DefinedSet{
			DefinedType: gobgpapi.DefinedType_PREFIX,
			Name:        customImportRejectSet,
			Prefixes:    prefixes,
		}
		return nrc.bgpServer.AddDefinedSet(context.Background(),
			&gobgpapi.AddDefinedSetRequest{DefinedSet: customImportRejectDefinedSet})
	}
	return nil
}

func (nrc *NetworkRoutingController) addiBGPPeersDefinedSet() (map[v1core.IPFamily][]string, error) {
	iBGPPeerCIDRs := make(map[v1core.IPFamily][]string)
	if !nrc.bgpEnableInternal {
		return iBGPPeerCIDRs, nil
	}

	// Get the current list of the nodes from the local cache
	nodes := nrc.nodeLister.List()
	for _, node := range nodes {
		nodeObj := node.(*v1core.Node)
		targetNode, err := utils.NewRemoteKRNode(nodeObj)
		if err != nil {
			klog.Errorf("Failed to find a node IP and therefore cannot add internal BGP Peer: %v", err)
			continue
		}
		if targetNode.GetPrimaryNodeIP().To4() != nil {
			iBGPPeerCIDRs[v1core.IPv4Protocol] = append(iBGPPeerCIDRs[v1core.IPv4Protocol],
				targetNode.GetPrimaryNodeIP().String()+"/32")
		} else {
			iBGPPeerCIDRs[v1core.IPv6Protocol] = append(iBGPPeerCIDRs[v1core.IPv6Protocol],
				targetNode.GetPrimaryNodeIP().String()+"/128")
		}
	}

	for family, setName := range map[v1core.IPFamily]string{
		v1core.IPv4Protocol: iBGPPeerSet,
		v1core.IPv6Protocol: iBGPPeerSetV6,
	} {
		currentDefinedSet, err := nrc.getDefinedSetFromGoBGP(setName, gobgpapi.DefinedType_NEIGHBOR)
		if err != nil {
			return iBGPPeerCIDRs, err
		}

		if currentDefinedSet == nil {
			iBGPPeerNS := &gobgpapi.DefinedSet{
				DefinedType: gobgpapi.DefinedType_NEIGHBOR,
				Name:        setName,
				List:        iBGPPeerCIDRs[family],
			}
			err = nrc.bgpServer.AddDefinedSet(context.Background(),
				&gobgpapi.AddDefinedSetRequest{DefinedSet: iBGPPeerNS})
			if err != nil {
				return iBGPPeerCIDRs, err
			}
			continue
		}

		currentCIDRs := currentDefinedSet.List
		sort.Strings(iBGPPeerCIDRs[family])
		sort.Strings(currentCIDRs)
		if reflect.DeepEqual(iBGPPeerCIDRs, currentCIDRs) {
			continue
		}
		toAdd := make([]string, 0)
		toDelete := make([]string, 0)
		for _, prefix := range iBGPPeerCIDRs[family] {
			add := true
			for _, currentPrefix := range currentDefinedSet.List {
				if prefix == currentPrefix {
					add = false
				}
			}
			if add {
				toAdd = append(toAdd, prefix)
			}
		}
		for _, currentPrefix := range currentDefinedSet.List {
			shouldDelete := true
			for _, prefix := range iBGPPeerCIDRs[family] {
				if currentPrefix == prefix {
					shouldDelete = false
				}
			}
			if shouldDelete {
				toDelete = append(toDelete, currentPrefix)
			}
		}
		iBGPPeerNS := &gobgpapi.DefinedSet{
			DefinedType: gobgpapi.DefinedType_NEIGHBOR,
			Name:        setName,
			List:        toAdd,
		}
		err = nrc.bgpServer.AddDefinedSet(context.Background(), &gobgpapi.AddDefinedSetRequest{DefinedSet: iBGPPeerNS})
		if err != nil {
			return iBGPPeerCIDRs, err
		}
		iBGPPeerNS = &gobgpapi.DefinedSet{
			DefinedType: gobgpapi.DefinedType_NEIGHBOR,
			Name:        setName,
			List:        toDelete,
		}
		err = nrc.bgpServer.DeleteDefinedSet(context.Background(),
			&gobgpapi.DeleteDefinedSetRequest{DefinedSet: iBGPPeerNS, All: false})
		if err != nil {
			return iBGPPeerCIDRs, err
		}
	}
	return iBGPPeerCIDRs, nil
}

func (nrc *NetworkRoutingController) addExternalBGPPeersDefinedSet() (map[v1core.IPFamily][]string, error) {

	externalBgpPeers := make([]string, 0)
	externalBGPPeerCIDRs := make(map[v1core.IPFamily][]string)

	if len(nrc.globalPeerRouters) > 0 {
		for _, peer := range nrc.globalPeerRouters {
			externalBgpPeers = append(externalBgpPeers, peer.Conf.NeighborAddress)
		}
	}
	if len(nrc.nodePeerRouters) > 0 {
		externalBgpPeers = append(externalBgpPeers, nrc.nodePeerRouters...)
	}
	if len(externalBgpPeers) == 0 {
		return externalBGPPeerCIDRs, nil
	}

	for family, extPeerSetName := range map[v1core.IPFamily]string{
		v1core.IPv4Protocol: externalPeerSet,
		v1core.IPv6Protocol: externalPeerSetV6} {
		currentDefinedSet, err := nrc.getDefinedSetFromGoBGP(extPeerSetName, gobgpapi.DefinedType_NEIGHBOR)
		if err != nil {
			return externalBGPPeerCIDRs, err
		}

		for _, peer := range externalBgpPeers {
			ip := net.ParseIP(peer)
			if ip == nil {
				klog.Warningf("wasn't able to parse the IP of peer: %s - skipping!", peer)
				continue
			}
			if ip.To4() != nil {
				// if we're not currently loading the IPv4 family, move on
				if family != v1core.IPv4Protocol {
					continue
				}
				externalBGPPeerCIDRs[family] = append(externalBGPPeerCIDRs[family], peer+"/32")
			} else if ip.To16() != nil {
				// if we're not currently loading the IPv6 family, move on
				if family != v1core.IPv6Protocol {
					continue
				}
				externalBGPPeerCIDRs[family] = append(externalBGPPeerCIDRs[family], peer+"/128")
			}
		}
		if currentDefinedSet == nil {
			eBGPPeerNS := &gobgpapi.DefinedSet{
				DefinedType: gobgpapi.DefinedType_NEIGHBOR,
				Name:        extPeerSetName,
				List:        externalBGPPeerCIDRs[family],
			}
			err = nrc.bgpServer.AddDefinedSet(context.Background(),
				&gobgpapi.AddDefinedSetRequest{DefinedSet: eBGPPeerNS})
			if err != nil {
				return externalBGPPeerCIDRs, err
			}
		}
	}

	return externalBGPPeerCIDRs, nil
}

// a slice of all peers is used as a match condition for reject statement of servicevipsdefinedset import policy
func (nrc *NetworkRoutingController) addAllBGPPeersDefinedSet(
	iBGPPeerCIDRs, externalBGPPeerCIDRs map[v1core.IPFamily][]string) error {

	for family, allPeerSetName := range map[v1core.IPFamily]string{
		v1core.IPv4Protocol: allPeerSet,
		v1core.IPv6Protocol: allPeerSetV6} {
		var currentDefinedSet *gobgpapi.DefinedSet
		currentDefinedSet, err := nrc.getDefinedSetFromGoBGP(allPeerSetName, gobgpapi.DefinedType_NEIGHBOR)
		if err != nil {
			return err
		}
		//nolint:gocritic // We intentionally append to a different array here to not change the passed
		// in externalBGPPeerCIDRs
		allBgpPeers := append(externalBGPPeerCIDRs[family], iBGPPeerCIDRs[family]...)
		if currentDefinedSet == nil {
			allPeerNS := &gobgpapi.DefinedSet{
				DefinedType: gobgpapi.DefinedType_NEIGHBOR,
				Name:        allPeerSetName,
				List:        allBgpPeers,
			}
			err = nrc.bgpServer.AddDefinedSet(context.Background(),
				&gobgpapi.AddDefinedSetRequest{DefinedSet: allPeerNS})
			if err != nil {
				return err
			}
			continue
		}

		toAdd := make([]string, 0)
		toDelete := make([]string, 0)
		for _, peer := range allBgpPeers {
			add := true
			for _, currentPeer := range currentDefinedSet.List {
				if peer == currentPeer {
					add = false
				}
			}
			if add {
				toAdd = append(toAdd, peer)
			}
		}
		for _, currentPeer := range currentDefinedSet.List {
			shouldDelete := true
			for _, peer := range allBgpPeers {
				if peer == currentPeer {
					shouldDelete = false
				}
			}
			if shouldDelete {
				toDelete = append(toDelete, currentPeer)
			}
		}
		allPeerNS := &gobgpapi.DefinedSet{
			DefinedType: gobgpapi.DefinedType_NEIGHBOR,
			Name:        allPeerSetName,
			List:        toAdd,
		}
		err = nrc.bgpServer.AddDefinedSet(context.Background(), &gobgpapi.AddDefinedSetRequest{DefinedSet: allPeerNS})
		if err != nil {
			return err
		}
		allPeerNS = &gobgpapi.DefinedSet{
			DefinedType: gobgpapi.DefinedType_NEIGHBOR,
			Name:        allPeerSetName,
			List:        toDelete,
		}
		err = nrc.bgpServer.DeleteDefinedSet(context.Background(),
			&gobgpapi.DeleteDefinedSetRequest{DefinedSet: allPeerNS, All: false})
		if err != nil {
			return err
		}
	}
	return nil
}

// BGP export policies are added so that following conditions are met:
//
//   - by default export of all routes from the RIB to the neighbour's is denied, and explicitly statements are added
//     to permit the desired routes to be exported
//   - each node is allowed to advertise its assigned pod CIDR's to all of its iBGP peer neighbours with same
//     ASN if --enable-ibgp=true
//   - each node is allowed to advertise its assigned pod CIDR's to all of its external BGP peer neighbours
//     only if --advertise-pod-cidr flag is set to true
//   - each node is NOT allowed to advertise its assigned pod CIDR's to all of its external BGP peer neighbours
//     only if --advertise-pod-cidr flag is set to false
//   - each node is allowed to advertise service VIP's (cluster ip, load balancer ip, external IP) ONLY to external
//     BGP peers
//   - each node is NOT allowed to advertise service VIP's (cluster ip, load balancer ip, external IP) to
//     iBGP peers
//   - an option to allow overriding the next-hop-address with the outgoing ip for external bgp peers
func (nrc *NetworkRoutingController) addExportPolicies() error {
	statementNames := make([]string, 0)

	var bgpActions gobgpapi.Actions
	if nrc.pathPrepend {
		prependAsn, err := strconv.ParseUint(nrc.pathPrependAS, 10, asnMaxBitSize)
		if err != nil {
			return errors.New("Invalid value for kube-router.io/path-prepend.as: " + err.Error())
		}
		bgpActions = gobgpapi.Actions{
			AsPrepend: &gobgpapi.AsPrependAction{
				Asn:    uint32(prependAsn),
				Repeat: uint32(nrc.pathPrependCount),
			},
		}
	}

	if nrc.bgpEnableInternal {
		actions := gobgpapi.Actions{
			RouteAction: gobgpapi.RouteAction_ACCEPT,
		}

		// statement to represent the export policy to permit advertising node's IPv4 & IPv6 pod CIDRs
		for _, podSet := range []string{podCIDRSet, podCIDRSetV6} {
			podSetEmpty, err := nrc.emptyCheckDefinedSets([]gobgpapi.ListDefinedSetRequest{
				{DefinedType: gobgpapi.DefinedType_PREFIX, Name: podSet},
			})
			if err != nil {
				return err
			}
			// if the set is empty, then skip it, so we don't have unintentional matches
			if podSetEmpty {
				continue
			}

			statement := gobgpapi.Statement{
				Conditions: &gobgpapi.Conditions{
					PrefixSet: &gobgpapi.MatchSet{
						Type: gobgpapi.MatchSet_ANY,
						Name: podSet,
					},
					NeighborSet: &gobgpapi.MatchSet{
						Type: gobgpapi.MatchSet_ANY,
						Name: iBGPPeerSet,
					},
				},
				Actions: &actions,
				Name:    podSet + iBGPPeerSet,
			}
			if err = nrc.ensureStatementExists(&statement); err != nil {
				return fmt.Errorf("could not check or create statement: %s - %v", statement.Name, err)
			}
			statementNames = append(statementNames, statement.Name)
		}
	}

	if len(nrc.globalPeerRouters) > 0 || len(nrc.nodePeerRouters) > 0 {

		bgpActions.RouteAction = gobgpapi.RouteAction_ACCEPT
		if nrc.overrideNextHop {
			bgpActions.Nexthop = &gobgpapi.NexthopAction{Self: true}
		}

		// set BGP communities for the routes advertised to peers for VIPs
		if len(nrc.nodeCommunities) > 0 {
			bgpActions.Community = &gobgpapi.CommunityAction{
				Type:        gobgpapi.CommunityAction_ADD,
				Communities: nrc.nodeCommunities,
			}
		}

		for _, peerSet := range []string{externalPeerSet, externalPeerSetV6} {
			peerSetEmpty, err := nrc.emptyCheckDefinedSets([]gobgpapi.ListDefinedSetRequest{
				{DefinedType: gobgpapi.DefinedType_NEIGHBOR, Name: peerSet},
			})
			if err != nil {
				return err
			}
			// if the set is empty, then skip it, so we don't have unintentional matches
			if peerSetEmpty {
				continue
			}

			for _, serviceVIPSet := range []string{serviceVIPsSet, serviceVIPsSetV6} {
				// statement to represent the export policy to permit advertising Service VIP's
				// only to the global BGP peer or node specific BGP peer
				vipSetEmpty, err := nrc.emptyCheckDefinedSets([]gobgpapi.ListDefinedSetRequest{
					{DefinedType: gobgpapi.DefinedType_PREFIX, Name: serviceVIPSet},
				})
				if err != nil {
					return err
				}
				// if the set is empty, then skip it, so we don't have unintentional matches
				if vipSetEmpty {
					continue
				}

				statement := gobgpapi.Statement{
					Conditions: &gobgpapi.Conditions{
						PrefixSet: &gobgpapi.MatchSet{
							Type: gobgpapi.MatchSet_ANY,
							Name: serviceVIPSet,
						},
						NeighborSet: &gobgpapi.MatchSet{
							Type: gobgpapi.MatchSet_ANY,
							Name: peerSet,
						},
					},
					Actions: &bgpActions,
					Name:    serviceVIPSet + peerSet,
				}
				if err = nrc.ensureStatementExists(&statement); err != nil {
					return fmt.Errorf("could not check or create statement: %s - %v", statement.Name, err)
				}
				statementNames = append(statementNames, statement.Name)
			}

			if nrc.advertisePodCidr {
				for _, podSet := range []string{podCIDRSet, podCIDRSetV6} {
					// if we are configured to advertise POD CIDRs then add export policies for all of our IPv4 and IPv6
					// peers for all IPv4 and IPv6 POD CIDRs
					podCIDREmpty, err := nrc.emptyCheckDefinedSets([]gobgpapi.ListDefinedSetRequest{
						{DefinedType: gobgpapi.DefinedType_PREFIX, Name: podSet},
					})
					if err != nil {
						return err
					}
					// if the set is empty, then skip it, so we don't have unintentional matches
					if podCIDREmpty {
						continue
					}

					statement := gobgpapi.Statement{
						Conditions: &gobgpapi.Conditions{
							PrefixSet: &gobgpapi.MatchSet{
								Type: gobgpapi.MatchSet_ANY,
								Name: podSet,
							},
							NeighborSet: &gobgpapi.MatchSet{
								Type: gobgpapi.MatchSet_ANY,
								Name: peerSet,
							},
						},
						Actions: &bgpActions,
						Name:    podSet + peerSet,
					}
					if err = nrc.ensureStatementExists(&statement); err != nil {
						return fmt.Errorf("could not check or create statement: %s - %v", statement.Name, err)
					}
					statementNames = append(statementNames, statement.Name)
				}
			}
		}
	}

	newStatementsByName := make([]*gobgpapi.Statement, 0)
	for _, st := range statementNames {
		newStatementsByName = append(newStatementsByName, &gobgpapi.Statement{
			Name: st,
		})
	}
	newPolicy := gobgpapi.Policy{
		Name:       kubeRouterExportPolicy,
		Statements: newStatementsByName,
	}

	currentPolicyFound := false
	currentPolicyName := ""
	currentPolicySameAsNewPolicy := false
	checkExistingPolicy := func(existingPolicy *gobgpapi.Policy) {
		if strings.HasPrefix(existingPolicy.Name, kubeRouterExportPolicy) {
			currentPolicyFound = true
			currentPolicyName = existingPolicy.Name
			currentPolicySameAsNewPolicy = statementsEqualByName(existingPolicy.Statements, newPolicy.Statements)
		}
	}
	err := nrc.bgpServer.ListPolicy(context.Background(), &gobgpapi.ListPolicyRequest{}, checkExistingPolicy)
	if err != nil {
		return errors.New("Failed to verify if kube-router BGP export policy exists: " + err.Error())
	}

	// If this is the first time this is run and there is no current policy in GoBGP initialize it before sending it off
	// to incrementAndCreatePolicy so that the below method doesn't have to carry context about where it is called from
	if currentPolicyName == "" {
		currentPolicyName = kubeRouterExportPolicy + "0"
		klog.Infof("Did not match any existing policies, starting import policy with default name: %s",
			currentPolicyName)
	}

	if currentPolicySameAsNewPolicy {
		klog.V(1).Infof("Policies in %s are the same, nothing to do here", currentPolicyName)
		return nil
	}

	klog.Infof("Current policy does not appear to match new policy: %s - creating new policy",
		newPolicy.Name)
	if err = nrc.incrementAndCreatePolicy(currentPolicyName, &newPolicy); err != nil {
		return fmt.Errorf("could not increment and create new policy: %v", err)
	}

	klog.Infof("Ensuring that policy %s is assigned", newPolicy.Name)
	err = nrc.assignPolicyByName(&newPolicy, gobgpapi.PolicyDirection_EXPORT, gobgpapi.RouteAction_REJECT)
	if err != nil {
		return fmt.Errorf("could not assign policy by name: %v", err)
	}

	if currentPolicyFound {
		klog.Infof("Now that new policy %s has been created and assigned, removing previous policy %s",
			newPolicy.Name, currentPolicyName)
		if err = nrc.unassignAndRemovePolicy(currentPolicyName, gobgpapi.PolicyDirection_EXPORT,
			gobgpapi.RouteAction_REJECT); err != nil {
			return fmt.Errorf("could not clean up old policy: %v", err)
		}
	}

	return nil
}

// BGP import policies are added so that the following conditions are met:
//   - do not import Service VIPs advertised from any peers, instead each kube-router originates and injects
//     Service VIPs into local rib.
func (nrc *NetworkRoutingController) addImportPolicies() error {
	statementNames := make([]string, 0)

	actions := gobgpapi.Actions{
		RouteAction: gobgpapi.RouteAction_REJECT,
	}
	for _, peerSet := range []string{allPeerSet, allPeerSetV6} {
		anyEmpty, err := nrc.emptyCheckDefinedSets([]gobgpapi.ListDefinedSetRequest{
			{DefinedType: gobgpapi.DefinedType_NEIGHBOR, Name: peerSet},
		})
		if err != nil {
			return err
		}
		// If set is empty, skip it so that we don't accidentally match VIPs against a blank set
		if anyEmpty {
			continue
		}
		for _, vipSet := range []string{serviceVIPsSet, serviceVIPsSetV6} {
			vipSetEmpty, err := nrc.emptyCheckDefinedSets([]gobgpapi.ListDefinedSetRequest{
				{DefinedType: gobgpapi.DefinedType_NEIGHBOR, Name: peerSet},
				{DefinedType: gobgpapi.DefinedType_PREFIX, Name: vipSet},
			})
			if err != nil {
				return err
			}
			// If either set is empty, skip it so that we don't accidentally match VIPs against a blank set
			if vipSetEmpty {
				continue
			}

			statement := gobgpapi.Statement{
				Conditions: &gobgpapi.Conditions{
					PrefixSet: &gobgpapi.MatchSet{
						Type: gobgpapi.MatchSet_ANY,
						Name: vipSet,
					},
					NeighborSet: &gobgpapi.MatchSet{
						Type: gobgpapi.MatchSet_ANY,
						Name: peerSet,
					},
				},
				Actions: &actions,
				Name:    vipSet + peerSet,
			}
			if err = nrc.ensureStatementExists(&statement); err != nil {
				return fmt.Errorf("could not check or create statement: %s - %v", statement.Name, err)
			}
			statementNames = append(statementNames, statement.Name)
		}

		statement := gobgpapi.Statement{
			Conditions: &gobgpapi.Conditions{
				PrefixSet: &gobgpapi.MatchSet{
					Type: gobgpapi.MatchSet_ANY,
					Name: defaultRouteSet,
				},
				NeighborSet: &gobgpapi.MatchSet{
					Type: gobgpapi.MatchSet_ANY,
					Name: peerSet,
				},
			},
			Actions: &actions,
			Name:    defaultRouteSet + peerSet,
		}
		if err = nrc.ensureStatementExists(&statement); err != nil {
			return fmt.Errorf("could not check or create statement: %s - %v", statement.Name, err)
		}
		statementNames = append(statementNames, statement.Name)

		if len(nrc.nodeCustomImportRejectIPNets) > 0 {
			statement = gobgpapi.Statement{
				Conditions: &gobgpapi.Conditions{
					PrefixSet: &gobgpapi.MatchSet{
						Name: customImportRejectSet,
						Type: gobgpapi.MatchSet_ANY,
					},
					NeighborSet: &gobgpapi.MatchSet{
						Type: gobgpapi.MatchSet_ANY,
						Name: peerSet,
					},
				},
				Actions: &actions,
				Name:    customImportRejectSet + peerSet,
			}
			if err = nrc.ensureStatementExists(&statement); err != nil {
				return fmt.Errorf("could not check or create statement: %s - %v", statement.Name, err)
			}
			statementNames = append(statementNames, statement.Name)
		}
	}

	newStatementsByName := make([]*gobgpapi.Statement, 0)
	for _, st := range statementNames {
		newStatementsByName = append(newStatementsByName, &gobgpapi.Statement{
			Name: st,
		})
	}
	newPolicy := gobgpapi.Policy{
		Name:       kubeRouterImportPolicy,
		Statements: newStatementsByName,
	}

	currentPolicyFound := false
	currentPolicyName := ""
	currentPolicySameAsNewPolicy := false
	checkExistingPolicy := func(existingPolicy *gobgpapi.Policy) {
		if strings.HasPrefix(existingPolicy.Name, kubeRouterImportPolicy) {
			currentPolicyFound = true
			currentPolicyName = existingPolicy.Name
			currentPolicySameAsNewPolicy = statementsEqualByName(existingPolicy.Statements, newPolicy.Statements)
		}
	}
	err := nrc.bgpServer.ListPolicy(context.Background(), &gobgpapi.ListPolicyRequest{}, checkExistingPolicy)
	if err != nil {
		return errors.New("Failed to verify if kube-router BGP export policy exists: " + err.Error())
	}

	// If this is the first time this is run and there is no current policy in GoBGP initialize it before sending it off
	// to incrementAndCreatePolicy so that the below method doesn't have to carry context about where it is called from
	if currentPolicyName == "" {
		currentPolicyName = kubeRouterImportPolicy + "0"
		klog.Infof("Did not match any existing policies, starting import policy with default name: %s",
			currentPolicyName)
	}

	if currentPolicySameAsNewPolicy {
		klog.V(1).Infof("Policies in %s are the same, nothing to do here", currentPolicyName)
		return nil
	}

	klog.Infof("Current policy does not appear to match new policy: %s - creating new policy",
		newPolicy.Name)
	if err = nrc.incrementAndCreatePolicy(currentPolicyName, &newPolicy); err != nil {
		return fmt.Errorf("could not increment and create new policy: %v", err)
	}

	klog.Infof("Ensuring that policy %s is assigned", newPolicy.Name)
	err = nrc.assignPolicyByName(&newPolicy, gobgpapi.PolicyDirection_IMPORT, gobgpapi.RouteAction_ACCEPT)
	if err != nil {
		return fmt.Errorf("could not assign policy by name: %v", err)
	}

	if currentPolicyFound {
		klog.Infof("Now that new policy %s has been created and assigned, removing previous policy %s",
			newPolicy.Name, currentPolicyName)
		if err = nrc.unassignAndRemovePolicy(currentPolicyName, gobgpapi.PolicyDirection_IMPORT,
			gobgpapi.RouteAction_ACCEPT); err != nil {
			return fmt.Errorf("could not clean up old policy: %v", err)
		}
	}

	return nil
}

// getDefinedSetFromGoBGP abstracts the logic for getting a DefinedSet object back for a given set type and name
func (nrc *NetworkRoutingController) getDefinedSetFromGoBGP(name string,
	defType gobgpapi.DefinedType) (defSet *gobgpapi.DefinedSet, err error) {
	err = nrc.bgpServer.ListDefinedSet(context.Background(),
		&gobgpapi.ListDefinedSetRequest{
			DefinedType: defType,
			Name:        name,
		}, func(ds *gobgpapi.DefinedSet) {
			defSet = ds
		})
	return
}

// emptyCheckDefinedSets Query requested sets from GoBGP to ensure that any of them aren't empty. Referencing an empty
// set in policy with MatchSet_ANY will cause it to match all items incorrectly and cause us to stop announcing
// important BGP paths, so we take extra precaution that we don't do this.
func (nrc *NetworkRoutingController) emptyCheckDefinedSets(defSets []gobgpapi.ListDefinedSetRequest) (bool, error) {
	for idx := range defSets {
		ds, err := nrc.getDefinedSetFromGoBGP(defSets[idx].Name, defSets[idx].DefinedType)
		if err != nil {
			return false, err
		}
		//nolint:exhaustive // We have a default here we don't need to exhaustively list all possible gobgpapi types
		switch defSets[idx].DefinedType {
		case gobgpapi.DefinedType_PREFIX:
			if len(ds.Prefixes) < 1 {
				return true, nil
			}
		default:
			if len(ds.List) < 1 {
				return true, nil
			}
		}
	}
	return false, nil
}

// ensureStatementExists checks to see if a statement already exists by looking for its name. If it already exists,
// method is a noop, if it does not exist, then we make sure to create it.
//
// NOTE: the checking in this method is done only by name. For now this works well because we don't change how
//
//	statements are formulated dynamically. If this changes in the future, then this method will need to be changed
func (nrc *NetworkRoutingController) ensureStatementExists(st *gobgpapi.Statement) error {
	found := false
	err := nrc.bgpServer.ListStatement(context.Background(), &gobgpapi.ListStatementRequest{},
		func(statement *gobgpapi.Statement) {
			if statement.Name == st.Name {
				found = true
			}
		})
	if err != nil {
		return fmt.Errorf("could not list statements: %v", err)
	}

	if found {
		return nil
	}

	err = nrc.bgpServer.AddStatement(context.Background(), &gobgpapi.AddStatementRequest{
		Statement: st,
	})
	if err != nil {
		return fmt.Errorf("could not add statement: %v", err)
	}

	return nil
}

func (nrc *NetworkRoutingController) incrementAndCreatePolicy(curPolName string,
	newPolicy *gobgpapi.Policy) error {
	// Get the current policy version and increment it
	polVer := 0
	polBaseName := ""
	var err error
	// sanity check
	if curPolName == "" {
		return fmt.Errorf("we require that curPolName be non-blank before calling this method")
	}

	matches := rePolicyVersionExtractor.FindStringSubmatch(curPolName)
	switch len(matches) {
	case 2:
		polBaseName = matches[1]
	case 3:
		polBaseName = matches[1]
		if polVer, err = strconv.Atoi(matches[2]); err != nil {
			return fmt.Errorf("failed to parse %s GoBGP policy name: %v", curPolName, err)
		}
	default:
		return fmt.Errorf("failed to parse %s GoBGP policy name via regex", curPolName)
	}
	newPolicyName := polBaseName + strconv.Itoa(polVer+1)
	newPolicy.Name = newPolicyName

	err = nrc.bgpServer.AddPolicy(context.Background(), &gobgpapi.AddPolicyRequest{
		Policy:                  newPolicy,
		ReferExistingStatements: true,
	})
	if err != nil {
		return errors.New("Failed to add policy: " + err.Error())
	}

	return nil
}

func (nrc *NetworkRoutingController) assignPolicyByName(newPolicy *gobgpapi.Policy,
	polDir gobgpapi.PolicyDirection, defaultAct gobgpapi.RouteAction) error {
	policyAssignmentExists := false

	// check to see if the policy is already assigned
	checkExistingPolicyAssignment := func(existingPolicyAssignment *gobgpapi.PolicyAssignment) {
		for _, policy := range existingPolicyAssignment.Policies {
			if policy.Name == newPolicy.Name {
				policyAssignmentExists = true
			}
		}
	}
	err := nrc.bgpServer.ListPolicyAssignment(context.Background(),
		&gobgpapi.ListPolicyAssignmentRequest{Name: "global", Direction: polDir},
		checkExistingPolicyAssignment)
	if err != nil {
		return errors.New("Failed to verify if kube-router BGP export policy assignment exists: " + err.Error())
	}
	if policyAssignmentExists {
		return nil
	}

	// policy is not assigned, assign it
	policyAssignment := gobgpapi.PolicyAssignment{
		Name:          "global",
		Direction:     polDir,
		Policies:      []*gobgpapi.Policy{newPolicy},
		DefaultAction: defaultAct,
	}
	err = nrc.bgpServer.AddPolicyAssignment(context.Background(),
		&gobgpapi.AddPolicyAssignmentRequest{Assignment: &policyAssignment})
	if err != nil {
		return errors.New("Failed to add policy assignment: " + err.Error())
	}

	return nil
}

func (nrc *NetworkRoutingController) unassignAndRemovePolicy(policyName string,
	polDir gobgpapi.PolicyDirection, defaultAct gobgpapi.RouteAction) error {
	err := nrc.bgpServer.DeletePolicyAssignment(context.Background(), &gobgpapi.DeletePolicyAssignmentRequest{
		Assignment: &gobgpapi.PolicyAssignment{
			Name:      "global",
			Direction: polDir,
			Policies: []*gobgpapi.Policy{
				{
					Name: policyName,
				},
			},
			DefaultAction: defaultAct,
		},
	})
	if err != nil {
		return fmt.Errorf("could not delete policy assignment for: %s - %v", policyName, err)
	}

	err = nrc.bgpServer.DeletePolicy(context.Background(), &gobgpapi.DeletePolicyRequest{
		Policy:             &gobgpapi.Policy{Name: policyName},
		PreserveStatements: true,
		All:                true, // All here tells GoBGP to delete the policy instead of just clearing the statements
	})
	if err != nil {
		return fmt.Errorf("could not delete policy for: %s - %v", policyName, err)
	}

	return nil
}