mirrors/kube-router
1
0
Fork 0
mirror of https://github.com/cloudnativelabs/kube-router.git synced 2025-11-18 19:41:05 +01:00
Code Issues Packages Projects Releases Wiki Activity

Routes Synchronization Routine

Browse Source
This commit is contained in:
RusoX89 2021-08-13 16:18:57 +01:00 committed by Aaron U'Ren
parent 492e0d126b
commit 23ac78cf94
5 changed files with 113 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/user-guide.md
View File

@ -64,6 +64,7 @@ Usage of kube-router:
--health-port uint16 Health check port, 0 = Disabled (default 20244) --health-port uint16 Health check port, 0 = Disabled (default 20244)
-h, --help Print usage information. -h, --help Print usage information.
--hostname-override string Overrides the NodeName of the node. Set this if kube-router is unable to determine your NodeName automatically. --hostname-override string Overrides the NodeName of the node. Set this if kube-router is unable to determine your NodeName automatically.
--injected-routes-sync-period duration The delay between route table synchronizations (e.g. '5s', '1m', '2h22m'). Must be greater than 0. (default 15s)
--iptables-sync-period duration The delay between iptables rule synchronizations (e.g. '5s', '1m'). Must be greater than 0. (default 5m0s) --iptables-sync-period duration The delay between iptables rule synchronizations (e.g. '5s', '1m'). Must be greater than 0. (default 5m0s)
--ipvs-graceful-period duration The graceful period before removing destinations from IPVS services (e.g. '5s', '1m', '2h22m'). Must be greater than 0. (default 30s) --ipvs-graceful-period duration The graceful period before removing destinations from IPVS services (e.g. '5s', '1m', '2h22m'). Must be greater than 0. (default 30s)
--ipvs-graceful-termination Enables the experimental IPVS graceful terminaton capability --ipvs-graceful-termination Enables the experimental IPVS graceful terminaton capability

36
pkg/controllers/routing/bgp_policies_test.go
View File

@ -36,16 +36,17 @@ func Test_AddPolicies(t *testing.T) {
{ {
"has nodes and services", "has nodes and services",
&NetworkRoutingController{ &NetworkRoutingController{
clientset: fake.NewSimpleClientset(), clientset: fake.NewSimpleClientset(),
hostnameOverride: "node-1", hostnameOverride: "node-1",
routerID: "10.0.0.0", routerID: "10.0.0.0",
bgpPort: 10000, bgpPort: 10000,
bgpFullMeshMode: false, bgpFullMeshMode: false,
bgpEnableInternal: true, bgpEnableInternal: true,
bgpServer: gobgp.NewBgpServer(), bgpServer: gobgp.NewBgpServer(),
activeNodes: make(map[string]bool), activeNodes: make(map[string]bool),
nodeAsnNumber: 100, nodeAsnNumber: 100,
podCidr: "172.20.0.0/24", podCidr: "172.20.0.0/24",
injectedRoutesSyncPeriod: 15,
}, },
[]*v1core.Node{ []*v1core.Node{
{ {
@ -195,7 +196,8 @@ func Test_AddPolicies(t *testing.T) {
}, },
}, },
}, },
nodeAsnNumber: 100, nodeAsnNumber: 100,
injectedRoutesSyncPeriod: 15,
}, },
[]*v1core.Node{ []*v1core.Node{
{ {
@ -366,7 +368,8 @@ func Test_AddPolicies(t *testing.T) {
}, },
}, },
}, },
nodeAsnNumber: 100, nodeAsnNumber: 100,
injectedRoutesSyncPeriod: 15,
}, },
[]*v1core.Node{ []*v1core.Node{
{ {
@ -520,7 +523,8 @@ func Test_AddPolicies(t *testing.T) {
}, },
}, },
}, },
nodeAsnNumber: 100, nodeAsnNumber: 100,
injectedRoutesSyncPeriod: 15,
}, },
[]*v1core.Node{ []*v1core.Node{
{ {
@ -697,7 +701,8 @@ func Test_AddPolicies(t *testing.T) {
}, },
}, },
}, },
nodeAsnNumber: 100, nodeAsnNumber: 100,
injectedRoutesSyncPeriod: 15,
}, },
[]*v1core.Node{ []*v1core.Node{
{ {
@ -874,7 +879,8 @@ func Test_AddPolicies(t *testing.T) {
}, },
}, },
}, },
nodeAsnNumber: 100, nodeAsnNumber: 100,
injectedRoutesSyncPeriod: 15,
}, },
[]*v1core.Node{ []*v1core.Node{
{ {

30
pkg/controllers/routing/network_routes_controller.go
View File

@ -132,6 +132,8 @@ type NetworkRoutingController struct {
podCidr string podCidr string
CNIFirewallSetup *sync.Cond CNIFirewallSetup *sync.Cond
ipsetMutex *sync.Mutex ipsetMutex *sync.Mutex
injectedRoutesSyncPeriod time.Duration
routeTableStateMap map[string]*netlink.Route
nodeLister cache.Indexer nodeLister cache.Indexer
svcLister cache.Indexer svcLister cache.Indexer
@ -142,6 +144,10 @@ type NetworkRoutingController struct {
EndpointsEventHandler cache.ResourceEventHandler EndpointsEventHandler cache.ResourceEventHandler
} }
func (nrc *NetworkRoutingController) addInjectedRoute(route *netlink.Route, dst *net.IPNet) {
nrc.routeTableStateMap[dst.String()] = route
}
// Run runs forever until we are notified on stop channel // Run runs forever until we are notified on stop channel
func (nrc *NetworkRoutingController) Run(healthChan chan<- *healthcheck.ControllerHeartbeat, stopCh <-chan struct{}, func (nrc *NetworkRoutingController) Run(healthChan chan<- *healthcheck.ControllerHeartbeat, stopCh <-chan struct{},
wg *sync.WaitGroup) { wg *sync.WaitGroup) {
@ -434,7 +440,26 @@ func (nrc *NetworkRoutingController) autoConfigureMTU() error {
return nil return nil
} }
func (nrc *NetworkRoutingController) watchRouteTable() {
go func() {
t := time.NewTicker(nrc.injectedRoutesSyncPeriod)
defer t.Stop()
for {
for _, route := range nrc.routeTableStateMap {
err := netlink.RouteReplace(route)
if err != nil {
klog.Errorf("Route could not be replaced due to : " + err.Error())
}
}
// Wait until the next iteration
<-t.C
}
}()
}
func (nrc *NetworkRoutingController) watchBgpUpdates() { func (nrc *NetworkRoutingController) watchBgpUpdates() {
// Start the route table watcher prior to adding anything
nrc.watchRouteTable()
pathWatch := func(path *gobgpapi.Path) { pathWatch := func(path *gobgpapi.Path) {
if nrc.MetricsEnabled { if nrc.MetricsEnabled {
metrics.ControllerBGPadvertisementsReceived.Inc() metrics.ControllerBGPadvertisementsReceived.Inc()
@ -622,7 +647,8 @@ func (nrc *NetworkRoutingController) injectRoute(path *gobgpapi.Path) error {
// Alright, everything is in place, and we have our route configured, let's add it to the host's routing table // Alright, everything is in place, and we have our route configured, let's add it to the host's routing table
klog.V(2).Infof("Inject route: '%s via %s' from peer to routing table", dst, nextHop) klog.V(2).Infof("Inject route: '%s via %s' from peer to routing table", dst, nextHop)
return netlink.RouteReplace(route) nrc.addInjectedRoute(route, dst)
return nil
} }
func (nrc *NetworkRoutingController) isPeerEstablished(peerIP string) (bool, error) { func (nrc *NetworkRoutingController) isPeerEstablished(peerIP string) (bool, error) {
@ -1155,6 +1181,7 @@ func NewNetworkRoutingController(clientset kubernetes.Interface,
nrc.peerMultihopTTL = kubeRouterConfig.PeerMultihopTTL nrc.peerMultihopTTL = kubeRouterConfig.PeerMultihopTTL
nrc.enablePodEgress = kubeRouterConfig.EnablePodEgress nrc.enablePodEgress = kubeRouterConfig.EnablePodEgress
nrc.syncPeriod = kubeRouterConfig.RoutesSyncPeriod nrc.syncPeriod = kubeRouterConfig.RoutesSyncPeriod
nrc.injectedRoutesSyncPeriod = kubeRouterConfig.InjectedRoutesSyncPeriod
nrc.overrideNextHop = kubeRouterConfig.OverrideNextHop nrc.overrideNextHop = kubeRouterConfig.OverrideNextHop
nrc.clientset = clientset nrc.clientset = clientset
nrc.activeNodes = make(map[string]bool) nrc.activeNodes = make(map[string]bool)
@ -1163,6 +1190,7 @@ func NewNetworkRoutingController(clientset kubernetes.Interface,
nrc.bgpServerStarted = false nrc.bgpServerStarted = false
nrc.disableSrcDstCheck = kubeRouterConfig.DisableSrcDstCheck nrc.disableSrcDstCheck = kubeRouterConfig.DisableSrcDstCheck
nrc.initSrcDstCheckDone = false nrc.initSrcDstCheckDone = false
nrc.routeTableStateMap = make(map[string]*netlink.Route)
nrc.bgpHoldtime = kubeRouterConfig.BGPHoldTime.Seconds() nrc.bgpHoldtime = kubeRouterConfig.BGPHoldTime.Seconds()
if nrc.bgpHoldtime > 65536 || nrc.bgpHoldtime < 3 { if nrc.bgpHoldtime > 65536 || nrc.bgpHoldtime < 3 {

110
pkg/controllers/routing/network_routes_controller_test.go
View File

@ -1530,15 +1530,16 @@ func Test_routeReflectorConfiguration(t *testing.T) {
{ {
"RR server with int cluster id", "RR server with int cluster id",
&NetworkRoutingController{ &NetworkRoutingController{
bgpFullMeshMode: false, bgpFullMeshMode: false,
bgpPort: 10000, bgpPort: 10000,
clientset: fake.NewSimpleClientset(), clientset: fake.NewSimpleClientset(),
nodeIP: net.ParseIP("10.0.0.0"), nodeIP: net.ParseIP("10.0.0.0"),
routerID: "10.0.0.0", routerID: "10.0.0.0",
bgpServer: gobgp.NewBgpServer(), bgpServer: gobgp.NewBgpServer(),
activeNodes: make(map[string]bool), activeNodes: make(map[string]bool),
nodeAsnNumber: 100, nodeAsnNumber: 100,
hostnameOverride: "node-1", hostnameOverride: "node-1",
injectedRoutesSyncPeriod: 15,
}, },
&v1core.Node{ &v1core.Node{
ObjectMeta: metav1.ObjectMeta{ ObjectMeta: metav1.ObjectMeta{
@ -1557,15 +1558,16 @@ func Test_routeReflectorConfiguration(t *testing.T) {
{ {
"RR server with IPv4 cluster id", "RR server with IPv4 cluster id",
&NetworkRoutingController{ &NetworkRoutingController{
bgpFullMeshMode: false, bgpFullMeshMode: false,
bgpPort: 10000, bgpPort: 10000,
clientset: fake.NewSimpleClientset(), clientset: fake.NewSimpleClientset(),
nodeIP: net.ParseIP("10.0.0.0"), nodeIP: net.ParseIP("10.0.0.0"),
routerID: "10.0.0.0", routerID: "10.0.0.0",
bgpServer: gobgp.NewBgpServer(), bgpServer: gobgp.NewBgpServer(),
activeNodes: make(map[string]bool), activeNodes: make(map[string]bool),
nodeAsnNumber: 100, nodeAsnNumber: 100,
hostnameOverride: "node-1", hostnameOverride: "node-1",
injectedRoutesSyncPeriod: 15,
}, },
&v1core.Node{ &v1core.Node{
ObjectMeta: metav1.ObjectMeta{ ObjectMeta: metav1.ObjectMeta{
@ -1584,15 +1586,16 @@ func Test_routeReflectorConfiguration(t *testing.T) {
{ {
"RR client with int cluster id", "RR client with int cluster id",
&NetworkRoutingController{ &NetworkRoutingController{
bgpFullMeshMode: false, bgpFullMeshMode: false,
bgpPort: 10000, bgpPort: 10000,
clientset: fake.NewSimpleClientset(), clientset: fake.NewSimpleClientset(),
nodeIP: net.ParseIP("10.0.0.0"), nodeIP: net.ParseIP("10.0.0.0"),
routerID: "10.0.0.0", routerID: "10.0.0.0",
bgpServer: gobgp.NewBgpServer(), bgpServer: gobgp.NewBgpServer(),
activeNodes: make(map[string]bool), activeNodes: make(map[string]bool),
nodeAsnNumber: 100, nodeAsnNumber: 100,
hostnameOverride: "node-1", hostnameOverride: "node-1",
injectedRoutesSyncPeriod: 15,
}, },
&v1core.Node{ &v1core.Node{
ObjectMeta: metav1.ObjectMeta{ ObjectMeta: metav1.ObjectMeta{
@ -1611,15 +1614,16 @@ func Test_routeReflectorConfiguration(t *testing.T) {
{ {
"RR client with IPv4 cluster id", "RR client with IPv4 cluster id",
&NetworkRoutingController{ &NetworkRoutingController{
bgpFullMeshMode: false, bgpFullMeshMode: false,
bgpPort: 10000, bgpPort: 10000,
clientset: fake.NewSimpleClientset(), clientset: fake.NewSimpleClientset(),
nodeIP: net.ParseIP("10.0.0.0"), nodeIP: net.ParseIP("10.0.0.0"),
routerID: "10.0.0.0", routerID: "10.0.0.0",
bgpServer: gobgp.NewBgpServer(), bgpServer: gobgp.NewBgpServer(),
activeNodes: make(map[string]bool), activeNodes: make(map[string]bool),
nodeAsnNumber: 100, nodeAsnNumber: 100,
hostnameOverride: "node-1", hostnameOverride: "node-1",
injectedRoutesSyncPeriod: 15,
}, },
&v1core.Node{ &v1core.Node{
ObjectMeta: metav1.ObjectMeta{ ObjectMeta: metav1.ObjectMeta{
@ -1638,14 +1642,15 @@ func Test_routeReflectorConfiguration(t *testing.T) {
{ {
"RR server with unparseable cluster id", "RR server with unparseable cluster id",
&NetworkRoutingController{ &NetworkRoutingController{
bgpFullMeshMode: false, bgpFullMeshMode: false,
bgpPort: 10000, bgpPort: 10000,
clientset: fake.NewSimpleClientset(), clientset: fake.NewSimpleClientset(),
nodeIP: net.ParseIP("10.0.0.0"), nodeIP: net.ParseIP("10.0.0.0"),
bgpServer: gobgp.NewBgpServer(), bgpServer: gobgp.NewBgpServer(),
activeNodes: make(map[string]bool), activeNodes: make(map[string]bool),
nodeAsnNumber: 100, nodeAsnNumber: 100,
hostnameOverride: "node-1", hostnameOverride: "node-1",
injectedRoutesSyncPeriod: 15,
}, },
&v1core.Node{ &v1core.Node{
ObjectMeta: metav1.ObjectMeta{ ObjectMeta: metav1.ObjectMeta{
@ -1664,14 +1669,15 @@ func Test_routeReflectorConfiguration(t *testing.T) {
{ {
"RR client with unparseable cluster id", "RR client with unparseable cluster id",
&NetworkRoutingController{ &NetworkRoutingController{
bgpFullMeshMode: false, bgpFullMeshMode: false,
bgpPort: 10000, bgpPort: 10000,
clientset: fake.NewSimpleClientset(), clientset: fake.NewSimpleClientset(),
nodeIP: net.ParseIP("10.0.0.0"), nodeIP: net.ParseIP("10.0.0.0"),
bgpServer: gobgp.NewBgpServer(), bgpServer: gobgp.NewBgpServer(),
activeNodes: make(map[string]bool), activeNodes: make(map[string]bool),
nodeAsnNumber: 100, nodeAsnNumber: 100,
hostnameOverride: "node-1", hostnameOverride: "node-1",
injectedRoutesSyncPeriod: 15,
}, },
&v1core.Node{ &v1core.Node{
ObjectMeta: metav1.ObjectMeta{ ObjectMeta: metav1.ObjectMeta{

4
pkg/options/options.go
View File

@ -42,6 +42,7 @@ type KubeRouterConfig struct {
HealthPort uint16 HealthPort uint16
HelpRequested bool HelpRequested bool
HostnameOverride string HostnameOverride string
InjectedRoutesSyncPeriod time.Duration
IPTablesSyncPeriod time.Duration IPTablesSyncPeriod time.Duration
IpvsGracefulPeriod time.Duration IpvsGracefulPeriod time.Duration
IpvsGracefulTermination bool IpvsGracefulTermination bool
@ -89,6 +90,7 @@ func NewKubeRouterConfig() *KubeRouterConfig {
NodePortRange: "30000-32767", NodePortRange: "30000-32767",
OverlayType: "subnet", OverlayType: "subnet",
RoutesSyncPeriod: 5 * time.Minute, RoutesSyncPeriod: 5 * time.Minute,
InjectedRoutesSyncPeriod: 15 * time.Second,
} }
} }
@ -149,6 +151,8 @@ func (s *KubeRouterConfig) AddFlags(fs *pflag.FlagSet) {
fs.StringVar(&s.HostnameOverride, "hostname-override", s.HostnameOverride, fs.StringVar(&s.HostnameOverride, "hostname-override", s.HostnameOverride,
"Overrides the NodeName of the node. Set this if kube-router is unable to determine your NodeName "+ "Overrides the NodeName of the node. Set this if kube-router is unable to determine your NodeName "+
"automatically.") "automatically.")
fs.DurationVar(&s.InjectedRoutesSyncPeriod, "injected-routes-sync-period", s.InjectedRoutesSyncPeriod,
"The delay between route table synchronizations (e.g. '5s', '1m', '2h22m'). Must be greater than 0.")
fs.DurationVar(&s.IPTablesSyncPeriod, "iptables-sync-period", s.IPTablesSyncPeriod, fs.DurationVar(&s.IPTablesSyncPeriod, "iptables-sync-period", s.IPTablesSyncPeriod,
"The delay between iptables rule synchronizations (e.g. '5s', '1m'). Must be greater than 0.") "The delay between iptables rule synchronizations (e.g. '5s', '1m'). Must be greater than 0.")
fs.DurationVar(&s.IpvsGracefulPeriod, "ipvs-graceful-period", s.IpvsGracefulPeriod, fs.DurationVar(&s.IpvsGracefulPeriod, "ipvs-graceful-period", s.IpvsGracefulPeriod,