diff --git a/docs/bgp.md b/docs/bgp.md index 6b3dddf5..6669d06b 100644 --- a/docs/bgp.md +++ b/docs/bgp.md @@ -37,10 +37,10 @@ get peered. ### Route-Reflector setup Without Full Mesh -This model support the common scheme of using Route Reflector Server node to concentrate +This model support the common scheme of using Route Reflector Server node to concentrate peering from Client Peer. This has the big advantage of not needing full mesh, and -scale better. In this mode kube-router expects each node is configured either in -Route Reflector server mode or in Route Reflector client mode. This is done +scale better. In this mode kube-router expects each node is configured either in +Route Reflector server mode or in Route Reflector client mode. This is done with node `kube-router.io/rr.server=ClusterID`, `kube-router.io/rr.client=ClusterId` respectively. In this mode each Route Reflector Client will only peer with Route Reflector Servers. Each Route Route Reflector Server will peer other Route Reflector @@ -52,7 +52,7 @@ Users can annotate node objects with the following command: kubectl annotate node "kube-router.io/rr.server=42" ``` -for Route Reflector server mode, and +for Route Reflector server mode, and ``` kubectl annotate node "kube-router.io/rr.client=42" @@ -92,6 +92,20 @@ kubectl annotate node "kube-router.io/peer.ips=192.168.1.99,192.168. kubectl annotate node "kube-router.io/peer.asns=65000,65000" ``` +### AS Path Prepending + +For traffic shaping purposes, you may want to prepend the AS path announced to peers. +This can be accomplished on a per-node basis with annotations: +- `kube-router.io/path-prepend.as` +- `kube-router.io/path-prepend.repeat-n` + +If you wanted to prepend all routes from a particular node with the AS 65000 five times, +you would run the following commands: +``` +kubectl annotate node "kube-router.io/path-prepend.as=65000" +kubectl annotate node "kube-router.io/path-prepend.repeat-n=5" +``` + ### BGP Peer Password Authentication The examples above have assumed there is no password authentication with BGP diff --git a/pkg/controllers/routing/export_policies.go b/pkg/controllers/routing/export_policies.go index 7246b949..24c2891a 100644 --- a/pkg/controllers/routing/export_policies.go +++ b/pkg/controllers/routing/export_policies.go @@ -66,6 +66,16 @@ func (nrc *NetworkRoutingController) addExportPolicies() error { statements := make([]config.Statement, 0) + var bgpActions config.BgpActions + if nrc.pathPrepend { + bgpActions = config.BgpActions{ + SetAsPathPrepend: config.SetAsPathPrepend{ + As: nrc.pathPrependAS, + RepeatN: nrc.pathPrependCount, + }, + } + } + if nrc.bgpEnableInternal { // Get the current list of the nodes from the local cache nodes := nrc.nodeLister.List() @@ -136,6 +146,7 @@ func (nrc *NetworkRoutingController) addExportPolicies() error { }, Actions: config.Actions{ RouteDisposition: config.ROUTE_DISPOSITION_ACCEPT_ROUTE, + BgpActions: bgpActions, }, }) if nrc.advertisePodCidr { diff --git a/pkg/controllers/routing/network_routes_controller.go b/pkg/controllers/routing/network_routes_controller.go index 6753313e..8b810e33 100644 --- a/pkg/controllers/routing/network_routes_controller.go +++ b/pkg/controllers/routing/network_routes_controller.go @@ -41,6 +41,8 @@ const ( nodeAddrsIPSetName = "kube-router-node-ips" nodeASNAnnotation = "kube-router.io/node.asn" + pathPrependASNAnnotation = "kube-router.io/path-prepend.as" + pathPrependRepeatNAnnotation = "kube-router.io/path-prepend.repeat-n" peerASNAnnotation = "kube-router.io/peer.asns" peerIPAnnotation = "kube-router.io/peer.ips" peerPasswordAnnotation = "kube-router.io/peer.passwords" @@ -87,6 +89,9 @@ type NetworkRoutingController struct { cniConfFile string initSrcDstCheckDone bool ec2IamAuthorized bool + pathPrependAS string + pathPrependCount uint8 + pathPrepend bool nodeLister cache.Indexer svcLister cache.Indexer @@ -610,6 +615,28 @@ func (nrc *NetworkRoutingController) startBgpServer() error { nrc.bgpRRClient = true } + if prependASN, okASN := node.ObjectMeta.Annotations[pathPrependASNAnnotation]; okASN { + prependRepeatN, okRepeatN := node.ObjectMeta.Annotations[pathPrependRepeatNAnnotation] + + if !okRepeatN { + return fmt.Errorf("Both %s and %s must be set", pathPrependASNAnnotation, pathPrependRepeatNAnnotation) + } + + _, err := strconv.ParseUint(prependASN, 0, 32) + if err != nil { + return errors.New("Failed to parse ASN number specified to prepend") + } + + repeatN, err := strconv.ParseUint(prependRepeatN, 0, 8) + if err != nil { + return errors.New("Failed to parse number of times ASN should be repeated") + } + + nrc.pathPrepend = true + nrc.pathPrependAS = prependASN + nrc.pathPrependCount = uint8(repeatN) + } + nrc.bgpServer = gobgp.NewBgpServer() go nrc.bgpServer.Serve() @@ -787,7 +814,6 @@ func NewNetworkRoutingController(clientset kubernetes.Interface, nrc.advertiseExternalIP = kubeRouterConfig.AdvertiseExternalIp nrc.advertiseLoadBalancerIP = kubeRouterConfig.AdvertiseLoadBalancerIp nrc.advertisePodCidr = kubeRouterConfig.AdvertiseNodePodCidr - nrc.enableOverlays = kubeRouterConfig.EnableOverlay // Convert ints to uint32s diff --git a/pkg/controllers/routing/network_routes_controller_test.go b/pkg/controllers/routing/network_routes_controller_test.go index 4a85cd26..0b8c78d9 100644 --- a/pkg/controllers/routing/network_routes_controller_test.go +++ b/pkg/controllers/routing/network_routes_controller_test.go @@ -1503,6 +1503,287 @@ func Test_addExportPolicies(t *testing.T) { }, nil, }, + { + "prepends AS with external peers", + &NetworkRoutingController{ + clientset: fake.NewSimpleClientset(), + hostnameOverride: "node-1", + bgpEnableInternal: true, + bgpFullMeshMode: false, + pathPrepend: true, + pathPrependCount: 5, + pathPrependAS: "65100", + bgpServer: gobgp.NewBgpServer(), + activeNodes: make(map[string]bool), + globalPeerRouters: []*config.NeighborConfig{ + { + NeighborAddress: "10.10.0.1", + }, + { + NeighborAddress: "10.10.0.2", + }, + }, + nodeAsnNumber: 100, + }, + []*v1core.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-1", + Annotations: map[string]string{ + "kube-router.io/node.asn": "100", + }, + }, + Status: v1core.NodeStatus{ + Addresses: []v1core.NodeAddress{ + { + Type: v1core.NodeInternalIP, + Address: "10.0.0.1", + }, + }, + }, + Spec: v1core.NodeSpec{ + PodCIDR: "172.20.0.0/24", + }, + }, + }, + []*v1core.Service{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "svc-1", + }, + Spec: v1core.ServiceSpec{ + Type: "ClusterIP", + ClusterIP: "10.0.0.1", + ExternalIPs: []string{"1.1.1.1"}, + }, + }, + }, + &config.DefinedSets{ + PrefixSets: []config.PrefixSet{ + { + PrefixSetName: "podcidrprefixset", + PrefixList: []config.Prefix{ + { + IpPrefix: "172.20.0.0/24", + MasklengthRange: "24..24", + }, + }, + }, + }, + NeighborSets: []config.NeighborSet{}, + TagSets: []config.TagSet{}, + BgpDefinedSets: config.BgpDefinedSets{}, + }, + &config.DefinedSets{ + PrefixSets: []config.PrefixSet{ + { + PrefixSetName: "clusteripprefixset", + PrefixList: []config.Prefix{ + { + IpPrefix: "1.1.1.1/32", + MasklengthRange: "32..32", + }, + { + IpPrefix: "10.0.0.1/32", + MasklengthRange: "32..32", + }, + }, + }, + }, + NeighborSets: []config.NeighborSet{}, + TagSets: []config.TagSet{}, + BgpDefinedSets: config.BgpDefinedSets{}, + }, + &config.DefinedSets{ + PrefixSets: []config.PrefixSet{}, + NeighborSets: []config.NeighborSet{ + { + NeighborSetName: "externalpeerset", + NeighborInfoList: []string{"10.10.0.1/32", "10.10.0.2/32"}, + }, + }, + TagSets: []config.TagSet{}, + BgpDefinedSets: config.BgpDefinedSets{}, + }, + []*config.Statement{ + { + Name: "kube_router_stmt0", + Conditions: config.Conditions{ + MatchPrefixSet: config.MatchPrefixSet{ + PrefixSet: "podcidrprefixset", + MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, + }, + MatchNeighborSet: config.MatchNeighborSet{ + NeighborSet: "iBGPpeerset", + MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, + }, + }, + Actions: config.Actions{ + RouteDisposition: config.ROUTE_DISPOSITION_ACCEPT_ROUTE, + }, + }, + { + Name: "kube_router_stmt1", + Conditions: config.Conditions{ + MatchPrefixSet: config.MatchPrefixSet{ + PrefixSet: "clusteripprefixset", + MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, + }, + MatchNeighborSet: config.MatchNeighborSet{ + NeighborSet: "externalpeerset", + MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, + }, + }, + Actions: config.Actions{ + RouteDisposition: config.ROUTE_DISPOSITION_ACCEPT_ROUTE, + BgpActions: config.BgpActions{ + SetAsPathPrepend: config.SetAsPathPrepend{ + As: "65100", + RepeatN: 5, + }, + }, + }, + }, + }, + nil, + }, + { + "only prepends AS when both node annotations are present", + &NetworkRoutingController{ + clientset: fake.NewSimpleClientset(), + hostnameOverride: "node-1", + bgpEnableInternal: true, + bgpFullMeshMode: false, + pathPrepend: false, + pathPrependAS: "65100", + bgpServer: gobgp.NewBgpServer(), + activeNodes: make(map[string]bool), + globalPeerRouters: []*config.NeighborConfig{ + { + NeighborAddress: "10.10.0.1", + }, + { + NeighborAddress: "10.10.0.2", + }, + }, + nodeAsnNumber: 100, + }, + []*v1core.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-1", + Annotations: map[string]string{ + "kube-router.io/node.asn": "100", + }, + }, + Status: v1core.NodeStatus{ + Addresses: []v1core.NodeAddress{ + { + Type: v1core.NodeInternalIP, + Address: "10.0.0.1", + }, + }, + }, + Spec: v1core.NodeSpec{ + PodCIDR: "172.20.0.0/24", + }, + }, + }, + []*v1core.Service{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "svc-1", + }, + Spec: v1core.ServiceSpec{ + Type: "ClusterIP", + ClusterIP: "10.0.0.1", + ExternalIPs: []string{"1.1.1.1"}, + }, + }, + }, + &config.DefinedSets{ + PrefixSets: []config.PrefixSet{ + { + PrefixSetName: "podcidrprefixset", + PrefixList: []config.Prefix{ + { + IpPrefix: "172.20.0.0/24", + MasklengthRange: "24..24", + }, + }, + }, + }, + NeighborSets: []config.NeighborSet{}, + TagSets: []config.TagSet{}, + BgpDefinedSets: config.BgpDefinedSets{}, + }, + &config.DefinedSets{ + PrefixSets: []config.PrefixSet{ + { + PrefixSetName: "clusteripprefixset", + PrefixList: []config.Prefix{ + { + IpPrefix: "1.1.1.1/32", + MasklengthRange: "32..32", + }, + { + IpPrefix: "10.0.0.1/32", + MasklengthRange: "32..32", + }, + }, + }, + }, + NeighborSets: []config.NeighborSet{}, + TagSets: []config.TagSet{}, + BgpDefinedSets: config.BgpDefinedSets{}, + }, + &config.DefinedSets{ + PrefixSets: []config.PrefixSet{}, + NeighborSets: []config.NeighborSet{ + { + NeighborSetName: "externalpeerset", + NeighborInfoList: []string{"10.10.0.1/32", "10.10.0.2/32"}, + }, + }, + TagSets: []config.TagSet{}, + BgpDefinedSets: config.BgpDefinedSets{}, + }, + []*config.Statement{ + { + Name: "kube_router_stmt0", + Conditions: config.Conditions{ + MatchPrefixSet: config.MatchPrefixSet{ + PrefixSet: "podcidrprefixset", + MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, + }, + MatchNeighborSet: config.MatchNeighborSet{ + NeighborSet: "iBGPpeerset", + MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, + }, + }, + Actions: config.Actions{ + RouteDisposition: config.ROUTE_DISPOSITION_ACCEPT_ROUTE, + }, + }, + { + Name: "kube_router_stmt1", + Conditions: config.Conditions{ + MatchPrefixSet: config.MatchPrefixSet{ + PrefixSet: "clusteripprefixset", + MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, + }, + MatchNeighborSet: config.MatchNeighborSet{ + NeighborSet: "externalpeerset", + MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, + }, + }, + Actions: config.Actions{ + RouteDisposition: config.ROUTE_DISPOSITION_ACCEPT_ROUTE, + }, + }, + }, + nil, + }, } for _, testcase := range testcases { @@ -1615,7 +1896,7 @@ func Test_addExportPolicies(t *testing.T) { for _, expectedStatement := range testcase.policyStatements { found := false for _, statement := range statements { - if statement.Equal(expectedStatement) { + if reflect.DeepEqual(statement, expectedStatement) { found = true } }