From e19f2a69c2807413a9b55fb205c3dc7c78a561d2 Mon Sep 17 00:00:00 2001 From: Bryan Zubrod Date: Sun, 24 Sep 2017 23:21:12 -0500 Subject: [PATCH] BGP peer password auth, consistent configurations (#164) * Add --peer-router-password option Also: - Consolodated NRC peer fields into a []config.NeighborConfig to store address, asn, and password for each peer. - BREAKING: --peer-router and --peer-asn flags now take slices rather than strings. * Add password auth node annotation for external peer * Update documentation New CLI flags and annotations Renamed ones as well * Consistent CLI flags, annotations, and peer config BGP configs now all accept multiple values and are treated consistently. Other refactoring was done as well. * Stop bgpserver on peering errors to avoid listener leak * Clarify BGP doc sections Fix some typos --- Documentation/README.md | 50 +-- Documentation/bgp.md | 111 +++++-- app/controllers/network_routes_controller.go | 297 ++++++++++++------ .../network_services_controller.go | 2 +- app/options/options.go | 21 +- ...ll-service-daemonset-advertise-routes.yaml | 4 +- 6 files changed, 321 insertions(+), 164 deletions(-) diff --git a/Documentation/README.md b/Documentation/README.md index f5cd1ba6..01587870 100644 --- a/Documentation/README.md +++ b/Documentation/README.md @@ -97,28 +97,30 @@ Also you can choose to run kube-router as agent running on each cluster node. Al ``` Usage of ./kube-router: - --advertise-cluster-ip Add Cluster IP to the RIB and advertise to peers. - --cleanup-config Cleanup iptables rules, ipvs, ipset configuration and exit. - --cluster-asn string ASN number under which cluster nodes will run iBGP. - --config-sync-period duration The delay between apiserver configuration synchronizations (e.g. '5s', '1m'). Must be greater than 0. (default 1m0s) - --enable-pod-egress SNAT traffic from Pods to destinations outside the cluster. (default true) - --enable-overlay When enable-overlay set to true, IP-in-IP tunneling is used for pod-to-pod networking across nodes in different subnets. When set to false no tunneling is used and routing infrastrcture is expected to route traffic for pod-to-pod networking across nodes in different subnets (default true) - --hairpin-mode Add iptable rules for every Service Endpoint to support hairpin traffic. --h, --help Print usage information. - --hostname-override string Overrides the NodeName of the node. Set this if kube-router is unable to determine your NodeName automatically. - --iptables-sync-period duration The delay between iptables rule synchronizations (e.g. '5s', '1m'). Must be greater than 0. (default 1m0s) - --ipvs-sync-period duration The delay between ipvs config synchronizations (e.g. '5s', '1m', '2h22m'). Must be greater than 0. (default 1m0s) - --kubeconfig string Path to kubeconfig file with authorization information (the master location is set by the master flag). - --masquerade-all SNAT all traffic to cluster IP/node port. - --master string The address of the Kubernetes API server (overrides any value in kubeconfig). - --nodes-full-mesh Each node in the cluster will setup BGP peering with rest of the nodes. (default true) - --peer-asn string ASN number of the BGP peer to which cluster nodes will advertise cluster ip and node's pod cidr. - --peer-router string Comma sepereated list of ip address of the external routers to which all nodes will peer and advertise the cluster ip and pod cidr's. - --routes-sync-period duration The delay between route updates and advertisements (e.g. '5s', '1m', '2h22m'). Must be greater than 0. (default 1m0s) - --run-firewall Enables Network Policy -- sets up iptables to provide ingress firewall for pods. (default true) - --run-router Enables Pod Networking -- Advertises and learns the routes to Pods via iBGP. (default true) - --run-service-proxy Enables Service Proxy -- sets up IPVS for Kubernetes Services. (default true) - --nodeport-bindon-all-ip For service of NodePort type create IPVS service that listens on all IP's of the node. (default false) + --advertise-cluster-ip Add Cluster IP to the RIB and advertise to peers. + --cleanup-config Cleanup iptables rules, ipvs, ipset configuration and exit. + --cluster-asn uint ASN number under which cluster nodes will run iBGP. + --cluster-cidr string CIDR range of pods in the cluster. It is used to identify traffic originating from and destinated to pods. + --config-sync-period duration The delay between apiserver configuration synchronizations (e.g. '5s', '1m'). Must be greater than 0. (default 1m0s) + --enable-overlay When enable-overlay set to true, IP-in-IP tunneling is used for pod-to-pod networking across nodes in different subnets. When set to false no tunneling is used and routing infrastrcture is expected to route traffic for pod-to-pod networking across nodes in different subnets (default true) + --enable-pod-egress SNAT traffic from Pods to destinations outside the cluster. (default true) + --hairpin-mode Add iptable rules for every Service Endpoint to support hairpin traffic. + -h, --help Print usage information. + --hostname-override string Overrides the NodeName of the node. Set this if kube-router is unable to determine your NodeName automatically. + --iptables-sync-period duration The delay between iptables rule synchronizations (e.g. '5s', '1m'). Must be greater than 0. (default 1m0s) + --ipvs-sync-period duration The delay between ipvs config synchronizations (e.g. '5s', '1m', '2h22m'). Must be greater than 0. (default 1m0s) + --kubeconfig string Path to kubeconfig file with authorization information (the master location is set by the master flag). + --masquerade-all SNAT all traffic to cluster IP/node port. + --master string The address of the Kubernetes API server (overrides any value in kubeconfig). + --nodeport-bindon-all-ip For service of NodePort type create IPVS service that listens on all IP's of the node. + --nodes-full-mesh Each node in the cluster will setup BGP peering with rest of the nodes. (default true) + --peer-router-asns uintSlice ASN numbers of the BGP peer to which cluster nodes will advertise cluster ip and node's pod cidr. (default []) + --peer-router-ips ipSlice The ip address of the external router to which all nodes will peer and advertise the cluster ip and pod cidr's. (default []) + --peer-router-passwords stringSlice Password for authenticating against the BGP peer defined with "--peer-router-ips". + --routes-sync-period duration The delay between route updates and advertisements (e.g. '5s', '1m', '2h22m'). Must be greater than 0. (default 1m0s) + --run-firewall Enables Network Policy -- sets up iptables to provide ingress firewall for pods. (default true) + --run-router Enables Pod Networking -- Advertises and learns the routes to Pods via iBGP. (default true) + --run-service-proxy Enables Service Proxy -- sets up IPVS for Kubernetes Services. (default true)``` ``` ### requirements @@ -178,7 +180,7 @@ and run kube-proxy with the configuration you have. Communication from a Pod that is behind a Service to its own ClusterIP:Port is not supported by default. However, It can be enabled per-service by adding the -`kube-router.io/hairpin-mode=` annotation, or for all Services in a cluster by +`io.kube-router.net.service.hairpin=` annotation, or for all Services in a cluster by passing the flag `--hairpin-mode=true` to kube-router. Additionally, the `hairpin_mode` sysctl option must be set to `1` for all veth @@ -207,7 +209,7 @@ Service ClusterIP if it is logging the source IP. To enable hairpin traffic for Service `my-service`: ``` -kubectl annotate service my-service 'kube-router.io/hairpin-mode=' +kubectl annotate service my-service "io.kube-router.net.service.hairpin=" ``` diff --git a/Documentation/bgp.md b/Documentation/bgp.md index bc95c308..c21d1edc 100644 --- a/Documentation/bgp.md +++ b/Documentation/bgp.md @@ -1,48 +1,103 @@ -## Configuring BGP Peers +# Configuring BGP Peers When kube-router is used to provide pod-to-pod networking, BGP is used to exchange routes across the nodes. Kube-router -provides flexible networking models to support different deployment (public vs private cloud, routable vs non-routable -pod IP's, service ip's etc) +provides flexible networking models to support different deployments (public vs private cloud, routable vs non-routable +pod IP's, service ip's etc). -### Full node-to-node mesh +## Peering Within The Cluster +### Full Node-To-Node Mesh -This is the default mode. All nodes in the clusters form iBGP peering relationship with rest of the nodes forming full -node-to-node mesh. Each node advertise the pod CIDR allocated to the nodes with peers (rest of the nodes in the cluster). -There is no configuration required in this mode. All the nodes in the cluster are associated with private ASN 64512 -implicitly (which can be configured with `--cluster-asn` flag). Users are transparent to use of iBGP. This mode is -suitable in public cloud environments or small cluster deployments. In this mode all the nodes are expected to be L2 adjacent. +This is the default mode. All nodes in the clusters form iBGP peering +relationship with rest of the nodes forming full node-to-node mesh. Each node +advertise the pod CIDR allocated to the nodes with peers (rest of the nodes in +the cluster). There is no configuration required in this mode. All the nodes in +the cluster are associated with private ASN 64512 implicitly (which can be +configured with `--cluster-asn` flag). Users are transparent to use of iBGP. +This mode is suitable in public cloud environments or small cluster deployments. +In this mode all the nodes are expected to be L2 adjacent. -### Node specific BGP peers +### Node-To-Node Peering Without Full Mesh -This model support more than a single AS per cluster to allow AS per rack or AS per node models. Nodes in the cluster -does not form full node-to-node mesh. Users has to explicitly select this mode by specifying `--nodes-full-mesh=false` -when launching kube-router. In this mode kube-router expects each node is configured with ASN number to be used for the -node from the nodes API object annoations. Kube-router will use the configured value for the key `net.kuberouter.nodeasn` -in the node object as the ASN number for the node. +This model support more than a single AS per cluster to allow AS per rack or AS +per node models. Nodes in the cluster does not form full node-to-node mesh. +Users has to explicitly select this mode by specifying `--nodes-full-mesh=false` +when launching kube-router. In this mode kube-router expects each node is +configured with an ASN number from the node's API object annoations. Kube-router +will use the node's `io.kube-router.net.node.asn` annotation value as the ASN +number for the node. -Users can annotate node object with below command +Users can annotate node objects with the following command: ``` -kubectl annotate node "net.kuberouter.nodeasn=64512”" +kubectl annotate node "io.kube-router.net.node.asn=64512" ``` -Only nodes with in same ASN form full mesh. Two nodes with different configured ASN never gets peered. +Only nodes with in same ASN form full mesh. Two nodes with different ASNs never +get peered. -### Global BGP Peer +## Peering Outside The Cluster +### Global External BGP Peers -An optional global BGP peer can be configured by specifying `--peer-asn` and `--peer-router` parameters. When configured -each node in the cluster forms a peer relationship with specified global peer. Pod cidr, cluster IP's get advertised to -the global BGP peer. For redundancy you can also configure more than one peer router by specifying comma seperated list -of BGP peers for `--peer-router` flag, like `--peer-router=192.168.1.99,192.168.1.100` +An optional global BGP peer can be configured by specifying `--peer-router-asns` +and `--peer-router-ips` parameters. When configured each node in the cluster +forms a peer relationship with specified global peer. Pod CIDR and Cluster IP's +get advertised to the global BGP peer. For redundancy you can also configure +more than one peer router by specifying a slice of BGP peers. -### Node specific BGP peer +For example: +``` +--peer-router-ips="192.168.1.99,192.168.1.100" +--peer-router-asns="65000,65000" +``` -Alternativley, each node can be configured with one or mode node specific BGP peer. Information regarding node specific BGP peer is -read from node API object annotations `net.kuberouter.node.bgppeer.address` and `net.kuberouter.node.bgppeer.asn`. +### Node Specific External BGP Peers + +Alternativley, each node can be configured with one or more node specific BGP +peers. Information regarding node specific BGP peer is read from node API object +annotations: +- `io.kube-router.net.peer.ips` +- `io.kube-router.net.peer.asns` For e.g users can annotate node object with below commands ``` -kubectl annotate node β€œnet.kuberouter.node.bgppeer.address=192.168.1.98,192.168.1.99” -kubectl annotate node "net.kuberouter.node.bgppeer.asn=64513”" +kubectl annotate node "io.kube-router.net.peer.ips=192.168.1.98,192.168.1.99" +kubectl annotate node "io.kube-router.net.peer.asns=64513" +``` + +### BGP Peer Password Authentication + +The examples above have assumed there is no password authentication with BGP +peer routers. If you need to use a password for peering, you can use the +`--peer-router-passwords` CLI flag or the `io.kube-router.net.peer.passwords` node +annotation. + +#### Base64 Encoding Passwords + +To ensure passwords are easily parsed, but not easily read by human eyes, +kube-router requires that they are encoded as base64. + +On a Linux or MacOS system you can encode your passwords on the command line: +``` +$ echo "SecurePassword" | base64 +U2VjdXJlUGFzc3dvcmQK +``` + +#### Password Configuration Examples + +In this CLI flag example the first router (192.168.1.99) uses a password, while +the second (192.168.1.100) does not. +``` +--peer-router-ips="192.168.1.99,192.168.1.100" +--peer-router-asns="65000,65000" +--peer-router-passwords="U2VjdXJlUGFzc3dvcmQK," +``` + +Note the comma indicating the end of the first password. + +Now here's the same example but configured as node annotations: +``` +kubectl annotate node "io.kube-router.net.peer.ips=192.168.1.99,192.168.1.100" +kubectl annotate node "io.kube-router.net.peer.asns=65000,65000" +kubectl annotate node "io.kube-router.net.peer.passwords=U2VjdXJlUGFzc3dvcmQK," ``` diff --git a/app/controllers/network_routes_controller.go b/app/controllers/network_routes_controller.go index 2477bbaf..43d78b5e 100644 --- a/app/controllers/network_routes_controller.go +++ b/app/controllers/network_routes_controller.go @@ -1,6 +1,7 @@ package controllers import ( + "encoding/base64" "errors" "fmt" "io/ioutil" @@ -49,9 +50,8 @@ type NetworkRoutingController struct { advertiseClusterIp bool defaultNodeAsnNumber uint32 nodeAsnNumber uint32 - globalPeerRouters []string + globalPeerRouters []*config.NeighborConfig nodePeerRouters []string - globalPeerAsnNumber uint32 bgpFullMeshMode bool podSubnetsIpSet *ipset.IPSet enableOverlays bool @@ -65,8 +65,8 @@ var ( ) const ( - clustetNieghboursSet = "clusterneighboursset" - podSubnetIpSetName = "kube-router-pod-subnets" + clustetNeighborsSet = "clusterneighborsset" + podSubnetIpSetName = "kube-router-pod-subnets" ) // Run runs forever until we are notified on stop channel @@ -277,7 +277,6 @@ func (nrc *NetworkRoutingController) watchBgpUpdates() { } func (nrc *NetworkRoutingController) advertiseRoute() error { - cidr, err := utils.GetPodCidrFromNodeSpec(nrc.clientset, nrc.hostnameOverride) if err != nil { return err @@ -290,11 +289,14 @@ func (nrc *NetworkRoutingController) advertiseRoute() error { bgp.NewPathAttributeOrigin(0), bgp.NewPathAttributeNextHop(nrc.nodeIP.String()), } + glog.Infof("Advertising route: '%s/%s via %s' to peers", subnet, strconv.Itoa(cidrLen), nrc.nodeIP.String()) + if _, err := nrc.bgpServer.AddPath("", []*table.Path{table.NewPath(nil, bgp.NewIPAddrPrefix(uint8(cidrLen), subnet), false, attrs, time.Now(), false)}); err != nil { return fmt.Errorf(err.Error()) } + return nil } @@ -313,6 +315,110 @@ func (nrc *NetworkRoutingController) getClusterIps() ([]string, error) { return clusterIpList, nil } +// Used for processing Annotations that may contain multiple items +// Pass this the string and the delimiter +func stringToSlice(s, d string) []string { + ss := make([]string, 0) + if strings.Contains(s, d) { + ss = strings.Split(s, d) + } else { + ss = append(ss, s) + } + return ss +} + +func stringSliceToIPs(s []string) ([]net.IP, error) { + ips := make([]net.IP, 0) + for _, ipString := range s { + ip := net.ParseIP(ipString) + if ip == nil { + return nil, fmt.Errorf("Could not parse \"%s\" as an IP.", ipString) + } + ips = append(ips, ip) + } + return ips, nil +} + +func stringSliceToUInt32(s []string) ([]uint32, error) { + ints := make([]uint32, 0) + for _, intString := range s { + newInt, err := strconv.ParseUint(intString, 0, 32) + if err != nil { + return nil, fmt.Errorf("Could not parse \"%s\" as an integer.", intString) + } + ints = append(ints, uint32(newInt)) + } + return ints, nil +} + +func stringSliceB64Decode(s []string) ([]string, error) { + ss := make([]string, 0) + for _, b64String := range s { + decoded, err := base64.StdEncoding.DecodeString(b64String) + if err != nil { + return nil, fmt.Errorf("Could not parse \"%s\" as a base64 encoded string.", + b64String) + } + ss = append(ss, string(decoded)) + } + return ss, nil +} + +// Does validation and returns neighbor configs +func newGlobalPeers(ips []net.IP, asns []uint32, passwords []string) ( + []*config.NeighborConfig, error) { + peers := make([]*config.NeighborConfig, 0) + + // Validations + if len(ips) != len(asns) { + return nil, errors.New("Invalid peer router config. " + + "The number of IPs and ASN numbers must be equal.") + } + + if len(ips) != len(passwords) && len(passwords) != 0 { + return nil, errors.New("Invalid peer router config. " + + "The number of passwords should either be zero, or one per peer router." + + " Use blank items if a router doesn't expect a password.\n" + + "Example: \"pass,,pass\" OR [\"pass\",\"\",\"pass\"].") + } + + for i := 0; i < len(ips); i++ { + if asns[i] > 65534 { + return nil, fmt.Errorf("Invalid ASN number \"%d\" for global BGP peer.", + asns[i]) + } + + peer := &config.NeighborConfig{ + NeighborAddress: ips[i].String(), + PeerAs: asns[i], + } + + if len(passwords) != 0 { + peer.AuthPassword = passwords[i] + } + + peers = append(peers, peer) + } + + return peers, nil +} + +func connectToPeers(server *gobgp.BgpServer, peerConfigs []*config.NeighborConfig) error { + for _, peerConfig := range peerConfigs { + n := &config.Neighbor{ + Config: *peerConfig, + } + err := server.AddNeighbor(n) + if err != nil { + return fmt.Errorf("Error peering with peer router "+ + "\"%s\" due to: %s", peerConfig.NeighborAddress, err) + } + glog.Infof("Successfully configured %s in ASN %v as BGP peer to the node", + peerConfig.NeighborAddress, peerConfig.PeerAs) + } + return nil +} + // AdvertiseClusterIp advertises the service cluster ip the configured peers func (nrc *NetworkRoutingController) AdvertiseClusterIp(clusterIp string) error { @@ -389,10 +495,14 @@ func (nrc *NetworkRoutingController) addExportPolicies() error { externalBgpPeers := make([]string, 0) if len(nrc.globalPeerRouters) != 0 { - externalBgpPeers = append(externalBgpPeers, nrc.globalPeerRouters...) + for _, peer := range nrc.globalPeerRouters { + externalBgpPeers = append(externalBgpPeers, peer.NeighborAddress) + } } if len(nrc.nodePeerRouters) != 0 { - externalBgpPeers = append(externalBgpPeers, nrc.nodePeerRouters...) + for _, peer := range nrc.nodePeerRouters { + externalBgpPeers = append(externalBgpPeers, peer) + } } if len(externalBgpPeers) > 0 { ns, _ := table.NewNeighborSet(config.NeighborSet{ @@ -661,24 +771,27 @@ func (nrc *NetworkRoutingController) syncPeers() { continue } - // if node full mesh is not requested then just peer with nodes with same ASN (run iBGP among same ASN peers) + // if node full mesh is not requested then just peer with nodes with same ASN + // (run iBGP among same ASN peers) if !nrc.bgpFullMeshMode { - // if the node is not annotated with ASN number or with invalid ASN skip peering - nodeasn, ok := node.ObjectMeta.Annotations["net.kuberouter.nodeasn"] + nodeasn, ok := node.ObjectMeta.Annotations["io.kube-router.net.node.asn"] if !ok { - glog.Infof("Not peering with the Node %s as ASN number of the node is unknown.", nodeIP.String()) + glog.Infof("Not peering with the Node %s as ASN number of the node is unknown.", + nodeIP.String()) continue } asnNo, err := strconv.ParseUint(nodeasn, 0, 32) if err != nil { - glog.Infof("Not peering with the Node %s as ASN number of the node is invalid.", nodeIP.String()) + glog.Infof("Not peering with the Node %s as ASN number of the node is invalid.", + nodeIP.String()) continue } - // if the nodes ASN number is different from ASN number of current node skipp peering + // if the nodes ASN number is different from ASN number of current node skip peering if nrc.nodeAsnNumber != uint32(asnNo) { - glog.Infof("Not peering with the Node %s as ASN number of the node is different.", nodeIP.String()) + glog.Infof("Not peering with the Node %s as ASN number of the node is different.", + nodeIP.String()) continue } } @@ -715,7 +828,7 @@ func (nrc *NetworkRoutingController) syncPeers() { } } - // delete the neighbor for the node that is removed + // delete the neighbor for the nodes that are removed for _, ip := range removedNodes { n := &config.Neighbor{ Config: config.NeighborConfig{ @@ -855,7 +968,6 @@ func (nrc *NetworkRoutingController) OnNodeUpdate(nodeUpdate *watchers.NodeUpdat } func (nrc *NetworkRoutingController) startBgpServer() error { - var nodeAsnNumber uint32 node, err := utils.GetNodeObject(nrc.clientset, nrc.hostnameOverride) if err != nil { @@ -865,10 +977,10 @@ func (nrc *NetworkRoutingController) startBgpServer() error { if nrc.bgpFullMeshMode { nodeAsnNumber = nrc.defaultNodeAsnNumber } else { - nodeasn, ok := node.ObjectMeta.Annotations["net.kuberouter.nodeasn"] + nodeasn, ok := node.ObjectMeta.Annotations["io.kube-router.net.node.asn"] if !ok { - return errors.New("Could not find ASN number for the node. Node need to be annotated with ASN number " + - "details to start BGP server.") + return errors.New("Could not find ASN number for the node. " + + "Node needs to be annotated with ASN number details to start BGP server.") } glog.Infof("Found ASN for the node to be %s from the node annotations", nodeasn) asnNo, err := strconv.ParseUint(nodeasn, 0, 32) @@ -909,72 +1021,69 @@ func (nrc *NetworkRoutingController) startBgpServer() error { go nrc.watchBgpUpdates() - // if the global routing peer is configured then peer with it - // else peer with node specific BGP peer - if len(nrc.globalPeerRouters) != 0 && nrc.globalPeerAsnNumber != 0 { - for _, peer := range nrc.globalPeerRouters { - n := &config.Neighbor{ - Config: config.NeighborConfig{ - NeighborAddress: peer, - PeerAs: nrc.globalPeerAsnNumber, - }, - } - if err := nrc.bgpServer.AddNeighbor(n); err != nil { - nrc.bgpServer.Stop() - return errors.New("Failed to peer with global peer router \"" + peer + "\" due to: " + err.Error()) - } - } - } else { - nodeBgpPeerAsn, ok := node.ObjectMeta.Annotations["net.kuberouter.node.bgppeer.asn"] + // If the global routing peer is configured then peer with it + // else attempt to get peers from node specific BGP annotations. + if len(nrc.globalPeerRouters) == 0 { + // Get Global Peer Router ASN configs + nodeBgpPeerAsnsAnnotation, ok := node.ObjectMeta.Annotations["io.kube-router.net.peer.asns"] if !ok { glog.Infof("Could not find BGP peer info for the node in the node annotations so skipping configuring peer.") return nil } - asnNo, err := strconv.ParseUint(nodeBgpPeerAsn, 0, 32) + + asnStrings := stringToSlice(nodeBgpPeerAsnsAnnotation, ",") + peerASNs, err := stringSliceToUInt32(asnStrings) if err != nil { nrc.bgpServer.Stop() - return errors.New("Failed to parse ASN number specified for the the node in the annotations") + return fmt.Errorf("Failed to parse node's Peer ASN Numbers Annotation: %s", err) } - peerAsnNo := uint32(asnNo) - nodeBgpPeersAnnotation, ok := node.ObjectMeta.Annotations["net.kuberouter.node.bgppeer.address"] + // Get Global Peer Router IP Address configs + nodeBgpPeersAnnotation, ok := node.ObjectMeta.Annotations["io.kube-router.net.peer.ips"] if !ok { glog.Infof("Could not find BGP peer info for the node in the node annotations so skipping configuring peer.") return nil } - nodePeerRouters := make([]string, 0) - if strings.Contains(nodeBgpPeersAnnotation, ",") { - ips := strings.Split(nodeBgpPeersAnnotation, ",") - for _, ip := range ips { - if net.ParseIP(ip) == nil { - nrc.bgpServer.Stop() - return errors.New("Invalid node BGP peer router ip in the annotation: " + ip) - } - } - nodePeerRouters = append(nodePeerRouters, ips...) - } else { - if net.ParseIP(nodeBgpPeersAnnotation) == nil { - nrc.bgpServer.Stop() - return errors.New("Invalid node BGP peer router ip: " + nodeBgpPeersAnnotation) - } - nodePeerRouters = append(nodePeerRouters, nodeBgpPeersAnnotation) + ipStrings := stringToSlice(nodeBgpPeersAnnotation, ",") + peerIPs, err := stringSliceToIPs(ipStrings) + if err != nil { + nrc.bgpServer.Stop() + return fmt.Errorf("Failed to parse node's Peer Addresses Annotation: %s", err) } - for _, peer := range nodePeerRouters { - glog.Infof("Node is configured to peer with %s in ASN %v from the node annotations", peer, peerAsnNo) - n := &config.Neighbor{ - Config: config.NeighborConfig{ - NeighborAddress: peer, - PeerAs: peerAsnNo, - }, - } - if err := nrc.bgpServer.AddNeighbor(n); err != nil { + + // Get Global Peer Router Password configs + peerPasswords := []string{} + nodeBGPPasswordsAnnotation, ok := node.ObjectMeta.Annotations["io.kube-router.net.peer.passwords"] + if !ok { + glog.Infof("Could not find BGP peer password info in the node's annotations. Assuming no passwords.") + } else { + passStrings := stringToSlice(nodeBGPPasswordsAnnotation, ",") + peerPasswords, err = stringSliceB64Decode(passStrings) + if err != nil { nrc.bgpServer.Stop() - return errors.New("Failed to peer with node specific BGP peer router: " + peer + " due to " + err.Error()) + return fmt.Errorf("Failed to parse node's Peer Passwords Annotation: %s", err) } } - nrc.nodePeerRouters = nodePeerRouters - glog.Infof("Successfully configured %s in ASN %v as BGP peer to the node", nodeBgpPeersAnnotation, peerAsnNo) + // Create and set Global Peer Router complete configs + nrc.globalPeerRouters, err = newGlobalPeers(peerIPs, peerASNs, peerPasswords) + if err != nil { + nrc.bgpServer.Stop() + return fmt.Errorf("Failed to process Global Peer Router configs: %s", err) + } + + nrc.nodePeerRouters = ipStrings + } + + if len(nrc.globalPeerRouters) != 0 { + err := connectToPeers(nrc.bgpServer, nrc.globalPeerRouters) + if err != nil { + nrc.bgpServer.Stop() + return fmt.Errorf("Failed to peer with Global Peer Router(s): %s", + err) + } + } else { + glog.Infof("No Global Peer Routers configured. Peering skipped.") } return nil @@ -1049,15 +1158,11 @@ func NewNetworkRoutingController(clientset *kubernetes.Clientset, nrc.podSubnetsIpSet = nil } - if len(kubeRouterConfig.ClusterAsn) != 0 { - asn, err := strconv.ParseUint(kubeRouterConfig.ClusterAsn, 0, 32) - if err != nil { - return nil, errors.New("Invalid cluster ASN: " + err.Error()) - } - if asn > 65534 || asn < 64512 { + if kubeRouterConfig.ClusterAsn != 0 { + if kubeRouterConfig.ClusterAsn > 65534 || kubeRouterConfig.ClusterAsn < 64512 { return nil, errors.New("Invalid ASN number for cluster ASN") } - nrc.defaultNodeAsnNumber = uint32(asn) + nrc.defaultNodeAsnNumber = uint32(kubeRouterConfig.ClusterAsn) } else { nrc.defaultNodeAsnNumber = 64512 // this magic number is first of the private ASN range, use it as default } @@ -1066,37 +1171,25 @@ func NewNetworkRoutingController(clientset *kubernetes.Clientset, nrc.enableOverlays = kubeRouterConfig.EnableOverlay - if (len(kubeRouterConfig.PeerRouter) != 0 && len(kubeRouterConfig.PeerAsn) == 0) || - (len(kubeRouterConfig.PeerRouter) == 0 && len(kubeRouterConfig.PeerAsn) != 0) { - return nil, errors.New("Either both or none of the params --peer-asn, --peer-router must be specified") + // Convert ints to uint32s + peerASNs := make([]uint32, 0) + for _, i := range kubeRouterConfig.PeerASNs { + peerASNs = append(peerASNs, uint32(i)) } - if len(kubeRouterConfig.PeerRouter) != 0 && len(kubeRouterConfig.PeerAsn) != 0 { - - if strings.Contains(kubeRouterConfig.PeerRouter, ",") { - ips := strings.Split(kubeRouterConfig.PeerRouter, ",") - for _, ip := range ips { - if net.ParseIP(ip) == nil { - return nil, errors.New("Invalid global BGP peer router ip: " + kubeRouterConfig.PeerRouter) - } - } - nrc.globalPeerRouters = append(nrc.globalPeerRouters, ips...) - - } else { - if net.ParseIP(kubeRouterConfig.PeerRouter) == nil { - return nil, errors.New("Invalid global BGP peer router ip: " + kubeRouterConfig.PeerRouter) - } - nrc.globalPeerRouters = append(nrc.globalPeerRouters, kubeRouterConfig.PeerRouter) - } - - asn, err := strconv.ParseUint(kubeRouterConfig.PeerAsn, 0, 32) + // Decode base64 passwords + peerPasswords := make([]string, 0) + if len(kubeRouterConfig.PeerPasswords) != 0 { + peerPasswords, err = stringSliceB64Decode(kubeRouterConfig.PeerPasswords) if err != nil { - return nil, errors.New("Invalid global BGP peer ASN: " + err.Error()) + return nil, fmt.Errorf("Failed to parse CLI Peer Passwords flag: %s", err) } - if asn > 65534 { - return nil, errors.New("Invalid ASN number for global BGP peer") - } - nrc.globalPeerAsnNumber = uint32(asn) + } + + nrc.globalPeerRouters, err = newGlobalPeers(kubeRouterConfig.PeerRouters, + peerASNs, peerPasswords) + if err != nil { + return nil, fmt.Errorf("Error processing Global Peer Router configs: %s", err) } nrc.hostnameOverride = kubeRouterConfig.HostnameOverride diff --git a/app/controllers/network_services_controller.go b/app/controllers/network_services_controller.go index 574e8339..40a5015e 100644 --- a/app/controllers/network_services_controller.go +++ b/app/controllers/network_services_controller.go @@ -426,7 +426,7 @@ func buildServicesInfo() serviceInfoMap { } svcInfo.sessionAffinity = (svc.Spec.SessionAffinity == "ClientIP") - _, svcInfo.hairpin = svc.ObjectMeta.Annotations["kube-router.io/hairpin-mode"] + _, svcInfo.hairpin = svc.ObjectMeta.Annotations["io.kube-router.net.service.hairpin"] svcId := generateServiceId(svc.Namespace, svc.Name, port.Name) serviceMap[svcId] = &svcInfo diff --git a/app/options/options.go b/app/options/options.go index 6b75a398..3e2fac6b 100755 --- a/app/options/options.go +++ b/app/options/options.go @@ -1,6 +1,7 @@ package options import ( + "net" "time" "github.com/spf13/pflag" @@ -23,13 +24,15 @@ type KubeRouterConfig struct { EnablePodEgress bool HostnameOverride string AdvertiseClusterIp bool - PeerRouter string - ClusterAsn string - PeerAsn string + PeerRouters []net.IP + PeerASNs []uint + ClusterAsn uint FullMeshMode bool GlobalHairpinMode bool NodePortBindOnAllIp bool EnableOverlay bool + PeerPasswords []string + // FullMeshPassword string } func NewKubeRouterConfig() *KubeRouterConfig { @@ -72,12 +75,12 @@ func (s *KubeRouterConfig) AddFlags(fs *pflag.FlagSet) { "The delay between route updates and advertisements (e.g. '5s', '1m', '2h22m'). Must be greater than 0.") fs.BoolVar(&s.AdvertiseClusterIp, "advertise-cluster-ip", false, "Add Cluster IP to the RIB and advertise to peers.") - fs.StringVar(&s.PeerRouter, "peer-router", s.PeerRouter, + fs.IPSliceVar(&s.PeerRouters, "peer-router-ips", s.PeerRouters, "The ip address of the external router to which all nodes will peer and advertise the cluster ip and pod cidr's.") - fs.StringVar(&s.ClusterAsn, "cluster-asn", s.ClusterAsn, + fs.UintVar(&s.ClusterAsn, "cluster-asn", s.ClusterAsn, "ASN number under which cluster nodes will run iBGP.") - fs.StringVar(&s.PeerAsn, "peer-asn", s.PeerAsn, - "ASN number of the BGP peer to which cluster nodes will advertise cluster ip and node's pod cidr.") + fs.UintSliceVar(&s.PeerASNs, "peer-router-asns", s.PeerASNs, + "ASN numbers of the BGP peer to which cluster nodes will advertise cluster ip and node's pod cidr.") fs.BoolVar(&s.FullMeshMode, "nodes-full-mesh", true, "Each node in the cluster will setup BGP peering with rest of the nodes.") fs.StringVar(&s.HostnameOverride, "hostname-override", s.HostnameOverride, @@ -89,4 +92,8 @@ func (s *KubeRouterConfig) AddFlags(fs *pflag.FlagSet) { fs.BoolVar(&s.EnableOverlay, "enable-overlay", true, "When enable-overlay set to true, IP-in-IP tunneling is used for pod-to-pod networking across nodes in different subnets. "+ "When set to false no tunneling is used and routing infrastrcture is expected to route traffic for pod-to-pod networking across nodes in different subnets") + fs.StringSliceVar(&s.PeerPasswords, "peer-router-passwords", s.PeerPasswords, + "Password for authenticating against the BGP peer defined with \"--peer-router-ips\".") + // fs.StringVar(&s.FullMeshPassword, "nodes-full-mesh-password", s.FullMeshPassword, + // "Password that cluster-node BGP servers will use to authenticate one another when \"--nodes-full-mesh\" is set.") } diff --git a/daemonset/kube-router-all-service-daemonset-advertise-routes.yaml b/daemonset/kube-router-all-service-daemonset-advertise-routes.yaml index 7118416a..7e19caaf 100644 --- a/daemonset/kube-router-all-service-daemonset-advertise-routes.yaml +++ b/daemonset/kube-router-all-service-daemonset-advertise-routes.yaml @@ -23,8 +23,8 @@ spec: - "--kubeconfig=/var/lib/kube-router/kubeconfig" - "--advertise-cluster-ip=true" - "--cluster-asn=64512" - - "--peer-router=192.168.1.99" - - "--peer-asn=64513" + - "--peer-router-ips=192.168.1.99" + - "--peer-router-asns=64513" securityContext: privileged: true imagePullPolicy: Always