diff --git a/docs/bgp.md b/docs/bgp.md index 88bb3c2e..88edaf27 100644 --- a/docs/bgp.md +++ b/docs/bgp.md @@ -97,16 +97,36 @@ kubectl annotate node "kube-router.io/peer.asns=65000,65000" For traffic shaping purposes, you may want to prepend the AS path announced to peers. This can be accomplished on a per-node basis with annotations: + - `kube-router.io/path-prepend.as` - `kube-router.io/path-prepend.repeat-n` If you wanted to prepend all routes from a particular node with the AS 65000 five times, you would run the following commands: + ``` kubectl annotate node "kube-router.io/path-prepend.as=65000" kubectl annotate node "kube-router.io/path-prepend.repeat-n=5" ``` +### BGP Peer Local IP configuration + +In some setups it might be desirable to set local IP address used for connectin external BGP +peers. This can be accomplished on nodes with annotations: + +- `kube-router.io/peer.localips` + +If set, this must be a list with a local IP address for each peer, or left empty to use nodeIP. + +Example: + +``` +kubectl annotate node "kube-router.io/peer.localips=10.1.1.1,10.1.1.2" +``` + +This will instruct kube-router to use IP `10.1.1.1` for first BGP peer as a local address, and use `10.1.1.2` +for the second. + ### BGP Peer Password Authentication The examples above have assumed there is no password authentication with BGP diff --git a/pkg/controllers/routing/bgp_peers.go b/pkg/controllers/routing/bgp_peers.go index 88ecad3d..e5184461 100644 --- a/pkg/controllers/routing/bgp_peers.go +++ b/pkg/controllers/routing/bgp_peers.go @@ -254,8 +254,8 @@ func (nrc *NetworkRoutingController) connectToExternalBGPPeers(server *gobgp.Bgp } // Does validation and returns neighbor configs -func newGlobalPeers(ips []net.IP, ports []uint32, asns []uint32, passwords []string, holdtime float64, - localAddress string) ([]*gobgpapi.Peer, error) { +func newGlobalPeers(ips []net.IP, ports []uint32, asns []uint32, passwords []string, localips []string, + holdtime float64, localAddress string) ([]*gobgpapi.Peer, error) { peers := make([]*gobgpapi.Peer, 0) // Validations @@ -275,6 +275,12 @@ func newGlobalPeers(ips []net.IP, ports []uint32, asns []uint32, passwords []str "Example: \"port,,port\" OR [\"port\",\"\",\"port\"]", strconv.Itoa(options.DefaultBgpPort)) } + if len(ips) != len(localips) && len(localips) != 0 { + return nil, fmt.Errorf("invalid peer router config. The number of localIPs should either be zero, or "+ + "one per peer router. If blank items are used, it will default to nodeIP, %s. "+ + "Example: \"10.1.1.1,,10.1.1.2\" OR [\"10.1.1.1\",\"\",\"10.1.1.2\"]", localAddress) + } + for i := 0; i < len(ips); i++ { if !((asns[i] >= 1 && asns[i] <= 23455) || (asns[i] >= 23457 && asns[i] <= 63999) || @@ -285,8 +291,7 @@ func newGlobalPeers(ips []net.IP, ports []uint32, asns []uint32, passwords []str asns[i]) } - // explicitly set neighbors.transport.config.local-address with nodeIP which is configured - // as their neighbor address at the remote peers. + // explicitly set neighbors.transport.config.local-address // this prevents the controller from initiating connection to its peers with a different IP address // when multiple L3 interfaces are active. peer := &gobgpapi.Peer{ @@ -309,6 +314,10 @@ func newGlobalPeers(ips []net.IP, ports []uint32, asns []uint32, passwords []str peer.Conf.AuthPassword = passwords[i] } + if len(localips) != 0 && localips[i] != "" { + peer.Transport.LocalAddress = localips[i] + } + peers = append(peers, peer) } diff --git a/pkg/controllers/routing/network_routes_controller.go b/pkg/controllers/routing/network_routes_controller.go index 6832e37d..0c7b2fa9 100644 --- a/pkg/controllers/routing/network_routes_controller.go +++ b/pkg/controllers/routing/network_routes_controller.go @@ -47,6 +47,7 @@ const ( pathPrependRepeatNAnnotation = "kube-router.io/path-prepend.repeat-n" peerASNAnnotation = "kube-router.io/peer.asns" peerIPAnnotation = "kube-router.io/peer.ips" + peerLocalIPAnnotation = "kube-router.io/peer.localips" //nolint:gosec // this is not a hardcoded password peerPasswordAnnotation = "kube-router.io/peer.passwords" peerPortAnnotation = "kube-router.io/peer.ports" @@ -1130,9 +1131,38 @@ func (nrc *NetworkRoutingController) startBgpServer(grpcServer bool) error { } } + // Get Global Peer Router LocalIP configs + var peerLocalIPs []string + nodeBGPPeerLocalIPs, ok := node.ObjectMeta.Annotations[peerLocalIPAnnotation] + if !ok { + klog.Infof("Could not find BGP peer local ip info in the node's annotations. Assuming node IP.") + } else { + peerLocalIPs = stringToSlice(nodeBGPPeerLocalIPs, ",") + err = func() error { + for _, s := range peerLocalIPs { + if s != "" { + ip := net.ParseIP(s) + if ip == nil { + return fmt.Errorf("could not parse \"%s\" as an IP", s) + } + } + } + + return nil + }() + if err != nil { + err2 := nrc.bgpServer.StopBgp(context.Background(), &gobgpapi.StopBgpRequest{}) + if err2 != nil { + klog.Errorf("Failed to stop bgpServer: %s", err2) + } + + return fmt.Errorf("failed to parse node's Peer Local Addresses Annotation: %s", err) + } + } + // Create and set Global Peer Router complete configs - nrc.globalPeerRouters, err = newGlobalPeers(peerIPs, peerPorts, peerASNs, peerPasswords, nrc.bgpHoldtime, - nrc.nodeIP.String()) + nrc.globalPeerRouters, err = newGlobalPeers(peerIPs, peerPorts, peerASNs, peerPasswords, peerLocalIPs, + nrc.bgpHoldtime, nrc.nodeIP.String()) if err != nil { err2 := nrc.bgpServer.StopBgp(context.Background(), &gobgpapi.StopBgpRequest{}) if err2 != nil { @@ -1327,7 +1357,7 @@ func NewNetworkRoutingController(clientset kubernetes.Interface, } nrc.globalPeerRouters, err = newGlobalPeers(kubeRouterConfig.PeerRouters, peerPorts, - peerASNs, peerPasswords, nrc.bgpHoldtime, nrc.nodeIP.String()) + peerASNs, peerPasswords, nil, nrc.bgpHoldtime, nrc.nodeIP.String()) if err != nil { return nil, fmt.Errorf("error processing Global Peer Router configs: %s", err) }