BGP peer password auth, consistent configurations (#164)

* Add --peer-router-password option
Also:
- Consolodated NRC peer fields into a []config.NeighborConfig
  to store address, asn, and password for each peer.
- BREAKING: --peer-router and --peer-asn flags now take slices
  rather than strings.

* Add password auth node annotation for external peer

* Update documentation

New CLI flags and annotations
Renamed ones as well

* Consistent CLI flags, annotations, and peer config

BGP configs now all accept multiple values and are treated consistently.
Other refactoring was done as well.

* Stop bgpserver on peering errors to avoid listener leak

* Clarify BGP doc sections

Fix some typos

Documentation/README.md

@@ -97,28 +97,30 @@ Also you can choose to run kube-router as agent running on each cluster node. Al
 
 ```
 Usage of ./kube-router:
-    --advertise-cluster-ip            Add Cluster IP to the RIB and advertise to peers.
+      --advertise-cluster-ip                Add Cluster IP to the RIB and advertise to peers.
-    --cleanup-config                  Cleanup iptables rules, ipvs, ipset configuration and exit.
+      --cleanup-config                      Cleanup iptables rules, ipvs, ipset configuration and exit.
-    --cluster-asn string              ASN number under which cluster nodes will run iBGP.
+      --cluster-asn uint                    ASN number under which cluster nodes will run iBGP.
-    --config-sync-period duration     The delay between apiserver configuration synchronizations (e.g. '5s', '1m').  Must be greater than 0. (default 1m0s)
+      --cluster-cidr string                 CIDR range of pods in the cluster. It is used to identify traffic originating from and destinated to pods.
-    --enable-pod-egress               SNAT traffic from Pods to destinations outside the cluster. (default true)
+      --config-sync-period duration         The delay between apiserver configuration synchronizations (e.g. '5s', '1m').  Must be greater than 0. (default 1m0s)
-    --enable-overlay                  When enable-overlay set to true, IP-in-IP tunneling is used for pod-to-pod networking across nodes in different subnets. When set to false no tunneling is used and routing infrastrcture is expected to route traffic for pod-to-pod networking across nodes in different subnets (default true)
+      --enable-overlay                      When enable-overlay set to true, IP-in-IP tunneling is used for pod-to-pod networking across nodes in different subnets. When set to false no tunneling is used and routing infrastrcture is expected to route traffic for pod-to-pod networking across nodes in different subnets (default true)
-    --hairpin-mode                    Add iptable rules for every Service Endpoint to support hairpin traffic.
+      --enable-pod-egress                   SNAT traffic from Pods to destinations outside the cluster. (default true)
--h, --help                            Print usage information.
+      --hairpin-mode                        Add iptable rules for every Service Endpoint to support hairpin traffic.
-    --hostname-override string        Overrides the NodeName of the node. Set this if kube-router is unable to determine your NodeName automatically.
+  -h, --help                                Print usage information.
-    --iptables-sync-period duration   The delay between iptables rule synchronizations (e.g. '5s', '1m'). Must be greater than 0. (default 1m0s)
+      --hostname-override string            Overrides the NodeName of the node. Set this if kube-router is unable to determine your NodeName automatically.
-    --ipvs-sync-period duration       The delay between ipvs config synchronizations (e.g. '5s', '1m', '2h22m'). Must be greater than 0. (default 1m0s)
+      --iptables-sync-period duration       The delay between iptables rule synchronizations (e.g. '5s', '1m'). Must be greater than 0. (default 1m0s)
-    --kubeconfig string               Path to kubeconfig file with authorization information (the master location is set by the master flag).
+      --ipvs-sync-period duration           The delay between ipvs config synchronizations (e.g. '5s', '1m', '2h22m'). Must be greater than 0. (default 1m0s)
-    --masquerade-all                  SNAT all traffic to cluster IP/node port.
+      --kubeconfig string                   Path to kubeconfig file with authorization information (the master location is set by the master flag).
-    --master string                   The address of the Kubernetes API server (overrides any value in kubeconfig).
+      --masquerade-all                      SNAT all traffic to cluster IP/node port.
-    --nodes-full-mesh                 Each node in the cluster will setup BGP peering with rest of the nodes. (default true)
+      --master string                       The address of the Kubernetes API server (overrides any value in kubeconfig).
-    --peer-asn string                 ASN number of the BGP peer to which cluster nodes will advertise cluster ip and node's pod cidr.
+      --nodeport-bindon-all-ip              For service of NodePort type create IPVS service that listens on all IP's of the node.
-    --peer-router string              Comma sepereated list of ip address of the external routers to which all nodes will peer and advertise the cluster ip and pod cidr's.
+      --nodes-full-mesh                     Each node in the cluster will setup BGP peering with rest of the nodes. (default true)
-    --routes-sync-period duration     The delay between route updates and advertisements (e.g. '5s', '1m', '2h22m'). Must be greater than 0. (default 1m0s)
+      --peer-router-asns uintSlice          ASN numbers of the BGP peer to which cluster nodes will advertise cluster ip and node's pod cidr. (default [])
-    --run-firewall                    Enables Network Policy -- sets up iptables to provide ingress firewall for pods. (default true)
+      --peer-router-ips ipSlice             The ip address of the external router to which all nodes will peer and advertise the cluster ip and pod cidr's. (default [])
-    --run-router                      Enables Pod Networking -- Advertises and learns the routes to Pods via iBGP. (default true)
+      --peer-router-passwords stringSlice   Password for authenticating against the BGP peer defined with "--peer-router-ips".
-    --run-service-proxy               Enables Service Proxy -- sets up IPVS for Kubernetes Services. (default true)
+      --routes-sync-period duration         The delay between route updates and advertisements (e.g. '5s', '1m', '2h22m'). Must be greater than 0. (default 1m0s)
-    --nodeport-bindon-all-ip          For service of NodePort type create IPVS service that listens on all IP's of the node. (default false)
+      --run-firewall                        Enables Network Policy -- sets up iptables to provide ingress firewall for pods. (default true)
+      --run-router                          Enables Pod Networking -- Advertises and learns the routes to Pods via iBGP. (default true)
+      --run-service-proxy                   Enables Service Proxy -- sets up IPVS for Kubernetes Services. (default true)```
 ```
 
 ### requirements
@@ -178,7 +180,7 @@ and run kube-proxy with the configuration you have.
 
 Communication from a Pod that is behind a Service to its own ClusterIP:Port is
 not supported by default.  However, It can be enabled per-service by adding the
-`kube-router.io/hairpin-mode=` annotation, or for all Services in a cluster by
+`io.kube-router.net.service.hairpin=` annotation, or for all Services in a cluster by
 passing the flag `--hairpin-mode=true` to kube-router.
 
 Additionally, the `hairpin_mode` sysctl option must be set to `1` for all veth
@@ -207,7 +209,7 @@ Service ClusterIP if it is logging the source IP.
 
 To enable hairpin traffic for Service `my-service`:
 ```
-kubectl annotate service my-service 'kube-router.io/hairpin-mode='
+kubectl annotate service my-service "io.kube-router.net.service.hairpin="
 ```
 
 

Documentation/bgp.md

@@ -1,48 +1,103 @@
-## Configuring BGP Peers
+# Configuring BGP Peers
 
 When kube-router is used to provide pod-to-pod networking, BGP is used to exchange routes across the nodes. Kube-router
-provides flexible networking models to support different deployment (public vs private cloud, routable vs non-routable 
+provides flexible networking models to support different deployments (public vs private cloud, routable vs non-routable
-pod IP's, service ip's etc) 
+pod IP's, service ip's etc).
 
-### Full node-to-node mesh
+## Peering Within The Cluster
+### Full Node-To-Node Mesh
 
-This is the default mode. All nodes in the clusters form iBGP peering relationship with rest of the nodes forming full 
+This is the default mode. All nodes in the clusters form iBGP peering
-node-to-node mesh. Each node advertise the pod CIDR allocated to the nodes with peers (rest of the nodes in the cluster). 
+relationship with rest of the nodes forming full node-to-node mesh. Each node
-There is no configuration required in this mode. All the nodes in the cluster are associated with private ASN 64512 
+advertise the pod CIDR allocated to the nodes with peers (rest of the nodes in
-implicitly (which can be configured with `--cluster-asn` flag). Users are transparent to use of iBGP. This mode is
+the cluster).  There is no configuration required in this mode. All the nodes in
-suitable in public cloud environments or small cluster deployments. In this mode all the nodes are expected to be L2 adjacent.
+the cluster are associated with private ASN 64512 implicitly (which can be
+configured with `--cluster-asn` flag). Users are transparent to use of iBGP.
+This mode is suitable in public cloud environments or small cluster deployments.
+In this mode all the nodes are expected to be L2 adjacent.
 
-### Node specific BGP peers
+### Node-To-Node Peering Without Full Mesh
 
-This model support more than a single AS per cluster to allow AS per rack or AS per node models. Nodes in the cluster
+This model support more than a single AS per cluster to allow AS per rack or AS
-does not form full node-to-node mesh. Users has to explicitly select this mode by specifying `--nodes-full-mesh=false` 
+per node models. Nodes in the cluster does not form full node-to-node mesh.
-when launching kube-router. In this mode kube-router expects each node is configured with ASN number to be used for the 
+Users has to explicitly select this mode by specifying `--nodes-full-mesh=false`
-node from the nodes API object annoations. Kube-router will use the configured value for the key `net.kuberouter.nodeasn`
+when launching kube-router. In this mode kube-router expects each node is
-in the node object as the ASN number for the node.
+configured with an ASN number from the node's API object annoations. Kube-router
+will use the node's `io.kube-router.net.node.asn` annotation value as the ASN
+number for the node.
 
-Users can annotate node object with below command
+Users can annotate node objects with the following command:
 
 ```
-kubectl annotate node <kube-node> "net.kuberouter.nodeasn=64512”"
+kubectl annotate node <kube-node> "io.kube-router.net.node.asn=64512"
 ```
 
-Only nodes with in same ASN form full mesh. Two nodes with different configured ASN never gets peered.
+Only nodes with in same ASN form full mesh. Two nodes with different ASNs never
+get peered.
 
-### Global BGP Peer
+## Peering Outside The Cluster
+### Global External BGP Peers
 
-An optional global BGP peer can be configured by specifying `--peer-asn` and `--peer-router` parameters. When configured
+An optional global BGP peer can be configured by specifying `--peer-router-asns`
-each node in the cluster forms a peer relationship with specified global peer. Pod cidr, cluster IP's get advertised to
+and `--peer-router-ips` parameters. When configured each node in the cluster
-the global BGP peer. For redundancy you can also configure more than one peer router by specifying comma seperated list
+forms a peer relationship with specified global peer. Pod CIDR and Cluster IP's
-of BGP peers for `--peer-router` flag, like `--peer-router=192.168.1.99,192.168.1.100`
+get advertised to the global BGP peer. For redundancy you can also configure
+more than one peer router by specifying a slice of BGP peers.
 
-### Node specific BGP peer
+For example:
+```
+--peer-router-ips="192.168.1.99,192.168.1.100"
+--peer-router-asns="65000,65000"
+```
 
-Alternativley, each node can be configured with one or mode node specific BGP peer. Information regarding node specific BGP peer is
+### Node Specific External BGP Peers
-read from node API object annotations `net.kuberouter.node.bgppeer.address` and `net.kuberouter.node.bgppeer.asn`.
+
+Alternativley, each node can be configured with one or more node specific BGP
+peers. Information regarding node specific BGP peer is read from node API object
+annotations:
+- `io.kube-router.net.peer.ips`
+- `io.kube-router.net.peer.asns`
 
 
 For e.g users can annotate node object with below commands
 ```
-kubectl annotate node <kube-node> “net.kuberouter.node.bgppeer.address=192.168.1.98,192.168.1.99”
+kubectl annotate node <kube-node> "io.kube-router.net.peer.ips=192.168.1.98,192.168.1.99"
-kubectl annotate node <kube-node> "net.kuberouter.node.bgppeer.asn=64513”"
+kubectl annotate node <kube-node> "io.kube-router.net.peer.asns=64513"
+```
+
+### BGP Peer Password Authentication
+
+The examples above have assumed there is no password authentication with BGP
+peer routers. If you need to use a password for peering, you can use the
+`--peer-router-passwords` CLI flag or the `io.kube-router.net.peer.passwords` node
+annotation.
+
+#### Base64 Encoding Passwords
+
+To ensure passwords are easily parsed, but not easily read by human eyes,
+kube-router requires that they are encoded as base64.
+
+On a Linux or MacOS system you can encode your passwords on the command line:
+```
+$ echo "SecurePassword" | base64
+U2VjdXJlUGFzc3dvcmQK
+```
+
+#### Password Configuration Examples
+
+In this CLI flag example the first router (192.168.1.99) uses a password, while
+the second (192.168.1.100) does not.
+```
+--peer-router-ips="192.168.1.99,192.168.1.100"
+--peer-router-asns="65000,65000"
+--peer-router-passwords="U2VjdXJlUGFzc3dvcmQK,"
+```
+
+Note the comma indicating the end of the first password.
+
+Now here's the same example but configured as node annotations:
+```
+kubectl annotate node <kube-node> "io.kube-router.net.peer.ips=192.168.1.99,192.168.1.100"
+kubectl annotate node <kube-node> "io.kube-router.net.peer.asns=65000,65000"
+kubectl annotate node <kube-node> "io.kube-router.net.peer.passwords=U2VjdXJlUGFzc3dvcmQK,"
 ```

app/controllers/network_routes_controller.go

@@ -1,6 +1,7 @@
 package controllers
 
 import (
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -49,9 +50,8 @@ type NetworkRoutingController struct {
 	advertiseClusterIp   bool
 	defaultNodeAsnNumber uint32
 	nodeAsnNumber        uint32
-	globalPeerRouters    []string
+	globalPeerRouters    []*config.NeighborConfig
 	nodePeerRouters      []string
-	globalPeerAsnNumber  uint32
 	bgpFullMeshMode      bool
 	podSubnetsIpSet      *ipset.IPSet
 	enableOverlays       bool
@@ -65,8 +65,8 @@ var (
 )
 
 const (
-	clustetNieghboursSet = "clusterneighboursset"
+	clustetNeighborsSet = "clusterneighborsset"
-	podSubnetIpSetName   = "kube-router-pod-subnets"
+	podSubnetIpSetName  = "kube-router-pod-subnets"
 )
 
 // Run runs forever until we are notified on stop channel
@@ -277,7 +277,6 @@ func (nrc *NetworkRoutingController) watchBgpUpdates() {
 }
 
 func (nrc *NetworkRoutingController) advertiseRoute() error {
-
 	cidr, err := utils.GetPodCidrFromNodeSpec(nrc.clientset, nrc.hostnameOverride)
 	if err != nil {
 		return err
@@ -290,11 +289,14 @@ func (nrc *NetworkRoutingController) advertiseRoute() error {
 		bgp.NewPathAttributeOrigin(0),
 		bgp.NewPathAttributeNextHop(nrc.nodeIP.String()),
 	}
+
 	glog.Infof("Advertising route: '%s/%s via %s' to peers", subnet, strconv.Itoa(cidrLen), nrc.nodeIP.String())
+
 	if _, err := nrc.bgpServer.AddPath("", []*table.Path{table.NewPath(nil, bgp.NewIPAddrPrefix(uint8(cidrLen),
 		subnet), false, attrs, time.Now(), false)}); err != nil {
 		return fmt.Errorf(err.Error())
 	}
+
 	return nil
 }
 
@@ -313,6 +315,110 @@ func (nrc *NetworkRoutingController) getClusterIps() ([]string, error) {
 	return clusterIpList, nil
 }
 
+// Used for processing Annotations that may contain multiple items
+// Pass this the string and the delimiter
+func stringToSlice(s, d string) []string {
+	ss := make([]string, 0)
+	if strings.Contains(s, d) {
+		ss = strings.Split(s, d)
+	} else {
+		ss = append(ss, s)
+	}
+	return ss
+}
+
+func stringSliceToIPs(s []string) ([]net.IP, error) {
+	ips := make([]net.IP, 0)
+	for _, ipString := range s {
+		ip := net.ParseIP(ipString)
+		if ip == nil {
+			return nil, fmt.Errorf("Could not parse \"%s\" as an IP.", ipString)
+		}
+		ips = append(ips, ip)
+	}
+	return ips, nil
+}
+
+func stringSliceToUInt32(s []string) ([]uint32, error) {
+	ints := make([]uint32, 0)
+	for _, intString := range s {
+		newInt, err := strconv.ParseUint(intString, 0, 32)
+		if err != nil {
+			return nil, fmt.Errorf("Could not parse \"%s\" as an integer.", intString)
+		}
+		ints = append(ints, uint32(newInt))
+	}
+	return ints, nil
+}
+
+func stringSliceB64Decode(s []string) ([]string, error) {
+	ss := make([]string, 0)
+	for _, b64String := range s {
+		decoded, err := base64.StdEncoding.DecodeString(b64String)
+		if err != nil {
+			return nil, fmt.Errorf("Could not parse \"%s\" as a base64 encoded string.",
+				b64String)
+		}
+		ss = append(ss, string(decoded))
+	}
+	return ss, nil
+}
+
+// Does validation and returns neighbor configs
+func newGlobalPeers(ips []net.IP, asns []uint32, passwords []string) (
+	[]*config.NeighborConfig, error) {
+	peers := make([]*config.NeighborConfig, 0)
+
+	// Validations
+	if len(ips) != len(asns) {
+		return nil, errors.New("Invalid peer router config. " +
+			"The number of IPs and ASN numbers must be equal.")
+	}
+
+	if len(ips) != len(passwords) && len(passwords) != 0 {
+		return nil, errors.New("Invalid peer router config. " +
+			"The number of passwords should either be zero, or one per peer router." +
+			" Use blank items if a router doesn't expect a password.\n" +
+			"Example: \"pass,,pass\" OR [\"pass\",\"\",\"pass\"].")
+	}
+
+	for i := 0; i < len(ips); i++ {
+		if asns[i] > 65534 {
+			return nil, fmt.Errorf("Invalid ASN number \"%d\" for global BGP peer.",
+				asns[i])
+		}
+
+		peer := &config.NeighborConfig{
+			NeighborAddress: ips[i].String(),
+			PeerAs:          asns[i],
+		}
+
+		if len(passwords) != 0 {
+			peer.AuthPassword = passwords[i]
+		}
+
+		peers = append(peers, peer)
+	}
+
+	return peers, nil
+}
+
+func connectToPeers(server *gobgp.BgpServer, peerConfigs []*config.NeighborConfig) error {
+	for _, peerConfig := range peerConfigs {
+		n := &config.Neighbor{
+			Config: *peerConfig,
+		}
+		err := server.AddNeighbor(n)
+		if err != nil {
+			return fmt.Errorf("Error peering with peer router "+
+				"\"%s\" due to: %s", peerConfig.NeighborAddress, err)
+		}
+		glog.Infof("Successfully configured %s in ASN %v as BGP peer to the node",
+			peerConfig.NeighborAddress, peerConfig.PeerAs)
+	}
+	return nil
+}
+
 // AdvertiseClusterIp  advertises the service cluster ip the configured peers
 func (nrc *NetworkRoutingController) AdvertiseClusterIp(clusterIp string) error {
 
@@ -389,10 +495,14 @@ func (nrc *NetworkRoutingController) addExportPolicies() error {
 
 	externalBgpPeers := make([]string, 0)
 	if len(nrc.globalPeerRouters) != 0 {
-		externalBgpPeers = append(externalBgpPeers, nrc.globalPeerRouters...)
+		for _, peer := range nrc.globalPeerRouters {
+			externalBgpPeers = append(externalBgpPeers, peer.NeighborAddress)
+		}
 	}
 	if len(nrc.nodePeerRouters) != 0 {
-		externalBgpPeers = append(externalBgpPeers, nrc.nodePeerRouters...)
+		for _, peer := range nrc.nodePeerRouters {
+			externalBgpPeers = append(externalBgpPeers, peer)
+		}
 	}
 	if len(externalBgpPeers) > 0 {
 		ns, _ := table.NewNeighborSet(config.NeighborSet{
@@ -661,24 +771,27 @@ func (nrc *NetworkRoutingController) syncPeers() {
 			continue
 		}
 
-		// if node full mesh is not requested then just peer with nodes with same ASN (run iBGP among same ASN peers)
+		// if node full mesh is not requested then just peer with nodes with same ASN
+		// (run iBGP among same ASN peers)
 		if !nrc.bgpFullMeshMode {
-			// if the node is not annotated with ASN number or with invalid ASN skip peering
+			nodeasn, ok := node.ObjectMeta.Annotations["io.kube-router.net.node.asn"]
-			nodeasn, ok := node.ObjectMeta.Annotations["net.kuberouter.nodeasn"]
 			if !ok {
-				glog.Infof("Not peering with the Node %s as ASN number of the node is unknown.", nodeIP.String())
+				glog.Infof("Not peering with the Node %s as ASN number of the node is unknown.",
+					nodeIP.String())
 				continue
 			}
 
 			asnNo, err := strconv.ParseUint(nodeasn, 0, 32)
 			if err != nil {
-				glog.Infof("Not peering with the Node %s as ASN number of the node is invalid.", nodeIP.String())
+				glog.Infof("Not peering with the Node %s as ASN number of the node is invalid.",
+					nodeIP.String())
 				continue
 			}
 
-			// if the nodes ASN number is different from ASN number of current node skipp peering
+			// if the nodes ASN number is different from ASN number of current node skip peering
 			if nrc.nodeAsnNumber != uint32(asnNo) {
-				glog.Infof("Not peering with the Node %s as ASN number of the node is different.", nodeIP.String())
+				glog.Infof("Not peering with the Node %s as ASN number of the node is different.",
+					nodeIP.String())
 				continue
 			}
 		}
@@ -715,7 +828,7 @@ func (nrc *NetworkRoutingController) syncPeers() {
 		}
 	}
 
-	// delete the neighbor for the node that is removed
+	// delete the neighbor for the nodes that are removed
 	for _, ip := range removedNodes {
 		n := &config.Neighbor{
 			Config: config.NeighborConfig{
@@ -855,7 +968,6 @@ func (nrc *NetworkRoutingController) OnNodeUpdate(nodeUpdate *watchers.NodeUpdat
 }
 
 func (nrc *NetworkRoutingController) startBgpServer() error {
-
 	var nodeAsnNumber uint32
 	node, err := utils.GetNodeObject(nrc.clientset, nrc.hostnameOverride)
 	if err != nil {
@@ -865,10 +977,10 @@ func (nrc *NetworkRoutingController) startBgpServer() error {
 	if nrc.bgpFullMeshMode {
 		nodeAsnNumber = nrc.defaultNodeAsnNumber
 	} else {
-		nodeasn, ok := node.ObjectMeta.Annotations["net.kuberouter.nodeasn"]
+		nodeasn, ok := node.ObjectMeta.Annotations["io.kube-router.net.node.asn"]
 		if !ok {
-			return errors.New("Could not find ASN number for the node. Node need to be annotated with ASN number " +
+			return errors.New("Could not find ASN number for the node. " +
-				"details to start BGP server.")
+				"Node needs to be annotated with ASN number details to start BGP server.")
 		}
 		glog.Infof("Found ASN for the node to be %s from the node annotations", nodeasn)
 		asnNo, err := strconv.ParseUint(nodeasn, 0, 32)
@@ -909,72 +1021,69 @@ func (nrc *NetworkRoutingController) startBgpServer() error {
 
 	go nrc.watchBgpUpdates()
 
-	// if the global routing peer is configured then peer with it
+	// If the global routing peer is configured then peer with it
-	// else peer with node specific BGP peer
+	// else attempt to get peers from node specific BGP annotations.
-	if len(nrc.globalPeerRouters) != 0 && nrc.globalPeerAsnNumber != 0 {
+	if len(nrc.globalPeerRouters) == 0 {
-		for _, peer := range nrc.globalPeerRouters {
+		// Get Global Peer Router ASN configs
-			n := &config.Neighbor{
+		nodeBgpPeerAsnsAnnotation, ok := node.ObjectMeta.Annotations["io.kube-router.net.peer.asns"]
-				Config: config.NeighborConfig{
-					NeighborAddress: peer,
-					PeerAs:          nrc.globalPeerAsnNumber,
-				},
-			}
-			if err := nrc.bgpServer.AddNeighbor(n); err != nil {
-				nrc.bgpServer.Stop()
-				return errors.New("Failed to peer with global peer router \"" + peer + "\" due to: " + err.Error())
-			}
-		}
-	} else {
-		nodeBgpPeerAsn, ok := node.ObjectMeta.Annotations["net.kuberouter.node.bgppeer.asn"]
 		if !ok {
 			glog.Infof("Could not find BGP peer info for the node in the node annotations so skipping configuring peer.")
 			return nil
 		}
-		asnNo, err := strconv.ParseUint(nodeBgpPeerAsn, 0, 32)
+
+		asnStrings := stringToSlice(nodeBgpPeerAsnsAnnotation, ",")
+		peerASNs, err := stringSliceToUInt32(asnStrings)
 		if err != nil {
 			nrc.bgpServer.Stop()
-			return errors.New("Failed to parse ASN number specified for the the node in the annotations")
+			return fmt.Errorf("Failed to parse node's Peer ASN Numbers Annotation: %s", err)
 		}
-		peerAsnNo := uint32(asnNo)
 
-		nodeBgpPeersAnnotation, ok := node.ObjectMeta.Annotations["net.kuberouter.node.bgppeer.address"]
+		// Get Global Peer Router IP Address configs
+		nodeBgpPeersAnnotation, ok := node.ObjectMeta.Annotations["io.kube-router.net.peer.ips"]
 		if !ok {
 			glog.Infof("Could not find BGP peer info for the node in the node annotations so skipping configuring peer.")
 			return nil
 		}
-		nodePeerRouters := make([]string, 0)
+		ipStrings := stringToSlice(nodeBgpPeersAnnotation, ",")
-		if strings.Contains(nodeBgpPeersAnnotation, ",") {
+		peerIPs, err := stringSliceToIPs(ipStrings)
-			ips := strings.Split(nodeBgpPeersAnnotation, ",")
+		if err != nil {
-			for _, ip := range ips {
+			nrc.bgpServer.Stop()
-				if net.ParseIP(ip) == nil {
+			return fmt.Errorf("Failed to parse node's Peer Addresses Annotation: %s", err)
-					nrc.bgpServer.Stop()
-					return errors.New("Invalid node BGP peer router ip in the annotation: " + ip)
-				}
-			}
-			nodePeerRouters = append(nodePeerRouters, ips...)
-		} else {
-			if net.ParseIP(nodeBgpPeersAnnotation) == nil {
-				nrc.bgpServer.Stop()
-				return errors.New("Invalid node BGP peer router ip: " + nodeBgpPeersAnnotation)
-			}
-			nodePeerRouters = append(nodePeerRouters, nodeBgpPeersAnnotation)
 		}
-		for _, peer := range nodePeerRouters {
+
-			glog.Infof("Node is configured to peer with %s in ASN %v from the node annotations", peer, peerAsnNo)
+		// Get Global Peer Router Password configs
-			n := &config.Neighbor{
+		peerPasswords := []string{}
-				Config: config.NeighborConfig{
+		nodeBGPPasswordsAnnotation, ok := node.ObjectMeta.Annotations["io.kube-router.net.peer.passwords"]
-					NeighborAddress: peer,
+		if !ok {
-					PeerAs:          peerAsnNo,
+			glog.Infof("Could not find BGP peer password info in the node's annotations. Assuming no passwords.")
-				},
+		} else {
-			}
+			passStrings := stringToSlice(nodeBGPPasswordsAnnotation, ",")
-			if err := nrc.bgpServer.AddNeighbor(n); err != nil {
+			peerPasswords, err = stringSliceB64Decode(passStrings)
+			if err != nil {
 				nrc.bgpServer.Stop()
-				return errors.New("Failed to peer with node specific BGP peer router: " + peer + " due to " + err.Error())
+				return fmt.Errorf("Failed to parse node's Peer Passwords Annotation: %s", err)
 			}
 		}
 
-		nrc.nodePeerRouters = nodePeerRouters
+		// Create and set Global Peer Router complete configs
-		glog.Infof("Successfully configured  %s in ASN %v as BGP peer to the node", nodeBgpPeersAnnotation, peerAsnNo)
+		nrc.globalPeerRouters, err = newGlobalPeers(peerIPs, peerASNs, peerPasswords)
+		if err != nil {
+			nrc.bgpServer.Stop()
+			return fmt.Errorf("Failed to process Global Peer Router configs: %s", err)
+		}
+
+		nrc.nodePeerRouters = ipStrings
+	}
+
+	if len(nrc.globalPeerRouters) != 0 {
+		err := connectToPeers(nrc.bgpServer, nrc.globalPeerRouters)
+		if err != nil {
+			nrc.bgpServer.Stop()
+			return fmt.Errorf("Failed to peer with Global Peer Router(s): %s",
+				err)
+		}
+	} else {
+		glog.Infof("No Global Peer Routers configured. Peering skipped.")
 	}
 
 	return nil
@@ -1049,15 +1158,11 @@ func NewNetworkRoutingController(clientset *kubernetes.Clientset,
 		nrc.podSubnetsIpSet = nil
 	}
 
-	if len(kubeRouterConfig.ClusterAsn) != 0 {
+	if kubeRouterConfig.ClusterAsn != 0 {
-		asn, err := strconv.ParseUint(kubeRouterConfig.ClusterAsn, 0, 32)
+		if kubeRouterConfig.ClusterAsn > 65534 || kubeRouterConfig.ClusterAsn < 64512 {
-		if err != nil {
-			return nil, errors.New("Invalid cluster ASN: " + err.Error())
-		}
-		if asn > 65534 || asn < 64512 {
 			return nil, errors.New("Invalid ASN number for cluster ASN")
 		}
-		nrc.defaultNodeAsnNumber = uint32(asn)
+		nrc.defaultNodeAsnNumber = uint32(kubeRouterConfig.ClusterAsn)
 	} else {
 		nrc.defaultNodeAsnNumber = 64512 // this magic number is first of the private ASN range, use it as default
 	}
@@ -1066,37 +1171,25 @@ func NewNetworkRoutingController(clientset *kubernetes.Clientset,
 
 	nrc.enableOverlays = kubeRouterConfig.EnableOverlay
 
-	if (len(kubeRouterConfig.PeerRouter) != 0 && len(kubeRouterConfig.PeerAsn) == 0) ||
+	// Convert ints to uint32s
-		(len(kubeRouterConfig.PeerRouter) == 0 && len(kubeRouterConfig.PeerAsn) != 0) {
+	peerASNs := make([]uint32, 0)
-		return nil, errors.New("Either both or none of the params --peer-asn, --peer-router must be specified")
+	for _, i := range kubeRouterConfig.PeerASNs {
+		peerASNs = append(peerASNs, uint32(i))
 	}
 
-	if len(kubeRouterConfig.PeerRouter) != 0 && len(kubeRouterConfig.PeerAsn) != 0 {
+	// Decode base64 passwords
-
+	peerPasswords := make([]string, 0)
-		if strings.Contains(kubeRouterConfig.PeerRouter, ",") {
+	if len(kubeRouterConfig.PeerPasswords) != 0 {
-			ips := strings.Split(kubeRouterConfig.PeerRouter, ",")
+		peerPasswords, err = stringSliceB64Decode(kubeRouterConfig.PeerPasswords)
-			for _, ip := range ips {
-				if net.ParseIP(ip) == nil {
-					return nil, errors.New("Invalid global BGP peer router ip: " + kubeRouterConfig.PeerRouter)
-				}
-			}
-			nrc.globalPeerRouters = append(nrc.globalPeerRouters, ips...)
-
-		} else {
-			if net.ParseIP(kubeRouterConfig.PeerRouter) == nil {
-				return nil, errors.New("Invalid global BGP peer router ip: " + kubeRouterConfig.PeerRouter)
-			}
-			nrc.globalPeerRouters = append(nrc.globalPeerRouters, kubeRouterConfig.PeerRouter)
-		}
-
-		asn, err := strconv.ParseUint(kubeRouterConfig.PeerAsn, 0, 32)
 		if err != nil {
-			return nil, errors.New("Invalid global BGP peer ASN: " + err.Error())
+			return nil, fmt.Errorf("Failed to parse CLI Peer Passwords flag: %s", err)
 		}
-		if asn > 65534 {
+	}
-			return nil, errors.New("Invalid ASN number for global BGP peer")
+
-		}
+	nrc.globalPeerRouters, err = newGlobalPeers(kubeRouterConfig.PeerRouters,
-		nrc.globalPeerAsnNumber = uint32(asn)
+		peerASNs, peerPasswords)
+	if err != nil {
+		return nil, fmt.Errorf("Error processing Global Peer Router configs: %s", err)
 	}
 
 	nrc.hostnameOverride = kubeRouterConfig.HostnameOverride

app/controllers/network_services_controller.go

@@ -426,7 +426,7 @@ func buildServicesInfo() serviceInfoMap {
 			}
 
 			svcInfo.sessionAffinity = (svc.Spec.SessionAffinity == "ClientIP")
-			_, svcInfo.hairpin = svc.ObjectMeta.Annotations["kube-router.io/hairpin-mode"]
+			_, svcInfo.hairpin = svc.ObjectMeta.Annotations["io.kube-router.net.service.hairpin"]
 
 			svcId := generateServiceId(svc.Namespace, svc.Name, port.Name)
 			serviceMap[svcId] = &svcInfo

app/options/options.go

@@ -1,6 +1,7 @@
 package options
 
 import (
+	"net"
 	"time"
 
 	"github.com/spf13/pflag"
@@ -23,13 +24,15 @@ type KubeRouterConfig struct {
 	EnablePodEgress     bool
 	HostnameOverride    string
 	AdvertiseClusterIp  bool
-	PeerRouter          string
+	PeerRouters         []net.IP
-	ClusterAsn          string
+	PeerASNs            []uint
-	PeerAsn             string
+	ClusterAsn          uint
 	FullMeshMode        bool
 	GlobalHairpinMode   bool
 	NodePortBindOnAllIp bool
 	EnableOverlay       bool
+	PeerPasswords       []string
+	// FullMeshPassword    string
 }
 
 func NewKubeRouterConfig() *KubeRouterConfig {
@@ -72,12 +75,12 @@ func (s *KubeRouterConfig) AddFlags(fs *pflag.FlagSet) {
 		"The delay between route updates and advertisements (e.g. '5s', '1m', '2h22m'). Must be greater than 0.")
 	fs.BoolVar(&s.AdvertiseClusterIp, "advertise-cluster-ip", false,
 		"Add Cluster IP to the RIB and advertise to peers.")
-	fs.StringVar(&s.PeerRouter, "peer-router", s.PeerRouter,
+	fs.IPSliceVar(&s.PeerRouters, "peer-router-ips", s.PeerRouters,
 		"The ip address of the external router to which all nodes will peer and advertise the cluster ip and pod cidr's.")
-	fs.StringVar(&s.ClusterAsn, "cluster-asn", s.ClusterAsn,
+	fs.UintVar(&s.ClusterAsn, "cluster-asn", s.ClusterAsn,
 		"ASN number under which cluster nodes will run iBGP.")
-	fs.StringVar(&s.PeerAsn, "peer-asn", s.PeerAsn,
+	fs.UintSliceVar(&s.PeerASNs, "peer-router-asns", s.PeerASNs,
-		"ASN number of the BGP peer to which cluster nodes will advertise cluster ip and node's pod cidr.")
+		"ASN numbers of the BGP peer to which cluster nodes will advertise cluster ip and node's pod cidr.")
 	fs.BoolVar(&s.FullMeshMode, "nodes-full-mesh", true,
 		"Each node in the cluster will setup BGP peering with rest of the nodes.")
 	fs.StringVar(&s.HostnameOverride, "hostname-override", s.HostnameOverride,
@@ -89,4 +92,8 @@ func (s *KubeRouterConfig) AddFlags(fs *pflag.FlagSet) {
 	fs.BoolVar(&s.EnableOverlay, "enable-overlay", true,
 		"When enable-overlay set to true, IP-in-IP tunneling is used for pod-to-pod networking across nodes in different subnets. "+
 			"When set to false no tunneling is used and routing infrastrcture is expected to route traffic for pod-to-pod networking across nodes in different subnets")
+	fs.StringSliceVar(&s.PeerPasswords, "peer-router-passwords", s.PeerPasswords,
+		"Password for authenticating against the BGP peer defined with \"--peer-router-ips\".")
+	// fs.StringVar(&s.FullMeshPassword, "nodes-full-mesh-password", s.FullMeshPassword,
+	// 	"Password that cluster-node BGP servers will use to authenticate one another when \"--nodes-full-mesh\" is set.")
 }

daemonset/kube-router-all-service-daemonset-advertise-routes.yaml

@@ -23,8 +23,8 @@ spec:
           - "--kubeconfig=/var/lib/kube-router/kubeconfig"
           - "--advertise-cluster-ip=true"
           - "--cluster-asn=64512"
-          - "--peer-router=192.168.1.99"
+          - "--peer-router-ips=192.168.1.99"
-          - "--peer-asn=64513"
+          - "--peer-router-asns=64513"
         securityContext:
           privileged: true
         imagePullPolicy: Always