diff --git a/docs/user-guide.md b/docs/user-guide.md index 62429d8c..095a5a2e 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -64,6 +64,7 @@ Usage of kube-router: --health-port uint16 Health check port, 0 = Disabled (default 20244) -h, --help Print usage information. --hostname-override string Overrides the NodeName of the node. Set this if kube-router is unable to determine your NodeName automatically. + --injected-routes-sync-period duration The delay between route table synchronizations (e.g. '5s', '1m', '2h22m'). Must be greater than 0. (default 15s) --iptables-sync-period duration The delay between iptables rule synchronizations (e.g. '5s', '1m'). Must be greater than 0. (default 5m0s) --ipvs-graceful-period duration The graceful period before removing destinations from IPVS services (e.g. '5s', '1m', '2h22m'). Must be greater than 0. (default 30s) --ipvs-graceful-termination Enables the experimental IPVS graceful terminaton capability diff --git a/pkg/controllers/routing/bgp_policies_test.go b/pkg/controllers/routing/bgp_policies_test.go index 9f1500e5..a6af6451 100644 --- a/pkg/controllers/routing/bgp_policies_test.go +++ b/pkg/controllers/routing/bgp_policies_test.go @@ -36,16 +36,17 @@ func Test_AddPolicies(t *testing.T) { { "has nodes and services", &NetworkRoutingController{ - clientset: fake.NewSimpleClientset(), - hostnameOverride: "node-1", - routerID: "10.0.0.0", - bgpPort: 10000, - bgpFullMeshMode: false, - bgpEnableInternal: true, - bgpServer: gobgp.NewBgpServer(), - activeNodes: make(map[string]bool), - nodeAsnNumber: 100, - podCidr: "172.20.0.0/24", + clientset: fake.NewSimpleClientset(), + hostnameOverride: "node-1", + routerID: "10.0.0.0", + bgpPort: 10000, + bgpFullMeshMode: false, + bgpEnableInternal: true, + bgpServer: gobgp.NewBgpServer(), + activeNodes: make(map[string]bool), + nodeAsnNumber: 100, + podCidr: "172.20.0.0/24", + injectedRoutesSyncPeriod: 15, }, []*v1core.Node{ { @@ -195,7 +196,8 @@ func Test_AddPolicies(t *testing.T) { }, }, }, - nodeAsnNumber: 100, + nodeAsnNumber: 100, + injectedRoutesSyncPeriod: 15, }, []*v1core.Node{ { @@ -366,7 +368,8 @@ func Test_AddPolicies(t *testing.T) { }, }, }, - nodeAsnNumber: 100, + nodeAsnNumber: 100, + injectedRoutesSyncPeriod: 15, }, []*v1core.Node{ { @@ -520,7 +523,8 @@ func Test_AddPolicies(t *testing.T) { }, }, }, - nodeAsnNumber: 100, + nodeAsnNumber: 100, + injectedRoutesSyncPeriod: 15, }, []*v1core.Node{ { @@ -697,7 +701,8 @@ func Test_AddPolicies(t *testing.T) { }, }, }, - nodeAsnNumber: 100, + nodeAsnNumber: 100, + injectedRoutesSyncPeriod: 15, }, []*v1core.Node{ { @@ -874,7 +879,8 @@ func Test_AddPolicies(t *testing.T) { }, }, }, - nodeAsnNumber: 100, + nodeAsnNumber: 100, + injectedRoutesSyncPeriod: 15, }, []*v1core.Node{ { diff --git a/pkg/controllers/routing/network_routes_controller.go b/pkg/controllers/routing/network_routes_controller.go index 917b4ff4..a785d54a 100644 --- a/pkg/controllers/routing/network_routes_controller.go +++ b/pkg/controllers/routing/network_routes_controller.go @@ -132,6 +132,8 @@ type NetworkRoutingController struct { podCidr string CNIFirewallSetup *sync.Cond ipsetMutex *sync.Mutex + injectedRoutesSyncPeriod time.Duration + routeTableStateMap map[string]*netlink.Route nodeLister cache.Indexer svcLister cache.Indexer @@ -142,6 +144,10 @@ type NetworkRoutingController struct { EndpointsEventHandler cache.ResourceEventHandler } +func (nrc *NetworkRoutingController) addInjectedRoute(route *netlink.Route, dst *net.IPNet) { + nrc.routeTableStateMap[dst.String()] = route +} + // Run runs forever until we are notified on stop channel func (nrc *NetworkRoutingController) Run(healthChan chan<- *healthcheck.ControllerHeartbeat, stopCh <-chan struct{}, wg *sync.WaitGroup) { @@ -434,7 +440,26 @@ func (nrc *NetworkRoutingController) autoConfigureMTU() error { return nil } +func (nrc *NetworkRoutingController) watchRouteTable() { + go func() { + t := time.NewTicker(nrc.injectedRoutesSyncPeriod) + defer t.Stop() + for { + for _, route := range nrc.routeTableStateMap { + err := netlink.RouteReplace(route) + if err != nil { + klog.Errorf("Route could not be replaced due to : " + err.Error()) + } + } + // Wait until the next iteration + <-t.C + } + }() +} + func (nrc *NetworkRoutingController) watchBgpUpdates() { + // Start the route table watcher prior to adding anything + nrc.watchRouteTable() pathWatch := func(path *gobgpapi.Path) { if nrc.MetricsEnabled { metrics.ControllerBGPadvertisementsReceived.Inc() @@ -622,7 +647,8 @@ func (nrc *NetworkRoutingController) injectRoute(path *gobgpapi.Path) error { // Alright, everything is in place, and we have our route configured, let's add it to the host's routing table klog.V(2).Infof("Inject route: '%s via %s' from peer to routing table", dst, nextHop) - return netlink.RouteReplace(route) + nrc.addInjectedRoute(route, dst) + return nil } func (nrc *NetworkRoutingController) isPeerEstablished(peerIP string) (bool, error) { @@ -1155,6 +1181,7 @@ func NewNetworkRoutingController(clientset kubernetes.Interface, nrc.peerMultihopTTL = kubeRouterConfig.PeerMultihopTTL nrc.enablePodEgress = kubeRouterConfig.EnablePodEgress nrc.syncPeriod = kubeRouterConfig.RoutesSyncPeriod + nrc.injectedRoutesSyncPeriod = kubeRouterConfig.InjectedRoutesSyncPeriod nrc.overrideNextHop = kubeRouterConfig.OverrideNextHop nrc.clientset = clientset nrc.activeNodes = make(map[string]bool) @@ -1163,6 +1190,7 @@ func NewNetworkRoutingController(clientset kubernetes.Interface, nrc.bgpServerStarted = false nrc.disableSrcDstCheck = kubeRouterConfig.DisableSrcDstCheck nrc.initSrcDstCheckDone = false + nrc.routeTableStateMap = make(map[string]*netlink.Route) nrc.bgpHoldtime = kubeRouterConfig.BGPHoldTime.Seconds() if nrc.bgpHoldtime > 65536 || nrc.bgpHoldtime < 3 { diff --git a/pkg/controllers/routing/network_routes_controller_test.go b/pkg/controllers/routing/network_routes_controller_test.go index db90661e..787047c4 100644 --- a/pkg/controllers/routing/network_routes_controller_test.go +++ b/pkg/controllers/routing/network_routes_controller_test.go @@ -1530,15 +1530,16 @@ func Test_routeReflectorConfiguration(t *testing.T) { { "RR server with int cluster id", &NetworkRoutingController{ - bgpFullMeshMode: false, - bgpPort: 10000, - clientset: fake.NewSimpleClientset(), - nodeIP: net.ParseIP("10.0.0.0"), - routerID: "10.0.0.0", - bgpServer: gobgp.NewBgpServer(), - activeNodes: make(map[string]bool), - nodeAsnNumber: 100, - hostnameOverride: "node-1", + bgpFullMeshMode: false, + bgpPort: 10000, + clientset: fake.NewSimpleClientset(), + nodeIP: net.ParseIP("10.0.0.0"), + routerID: "10.0.0.0", + bgpServer: gobgp.NewBgpServer(), + activeNodes: make(map[string]bool), + nodeAsnNumber: 100, + hostnameOverride: "node-1", + injectedRoutesSyncPeriod: 15, }, &v1core.Node{ ObjectMeta: metav1.ObjectMeta{ @@ -1557,15 +1558,16 @@ func Test_routeReflectorConfiguration(t *testing.T) { { "RR server with IPv4 cluster id", &NetworkRoutingController{ - bgpFullMeshMode: false, - bgpPort: 10000, - clientset: fake.NewSimpleClientset(), - nodeIP: net.ParseIP("10.0.0.0"), - routerID: "10.0.0.0", - bgpServer: gobgp.NewBgpServer(), - activeNodes: make(map[string]bool), - nodeAsnNumber: 100, - hostnameOverride: "node-1", + bgpFullMeshMode: false, + bgpPort: 10000, + clientset: fake.NewSimpleClientset(), + nodeIP: net.ParseIP("10.0.0.0"), + routerID: "10.0.0.0", + bgpServer: gobgp.NewBgpServer(), + activeNodes: make(map[string]bool), + nodeAsnNumber: 100, + hostnameOverride: "node-1", + injectedRoutesSyncPeriod: 15, }, &v1core.Node{ ObjectMeta: metav1.ObjectMeta{ @@ -1584,15 +1586,16 @@ func Test_routeReflectorConfiguration(t *testing.T) { { "RR client with int cluster id", &NetworkRoutingController{ - bgpFullMeshMode: false, - bgpPort: 10000, - clientset: fake.NewSimpleClientset(), - nodeIP: net.ParseIP("10.0.0.0"), - routerID: "10.0.0.0", - bgpServer: gobgp.NewBgpServer(), - activeNodes: make(map[string]bool), - nodeAsnNumber: 100, - hostnameOverride: "node-1", + bgpFullMeshMode: false, + bgpPort: 10000, + clientset: fake.NewSimpleClientset(), + nodeIP: net.ParseIP("10.0.0.0"), + routerID: "10.0.0.0", + bgpServer: gobgp.NewBgpServer(), + activeNodes: make(map[string]bool), + nodeAsnNumber: 100, + hostnameOverride: "node-1", + injectedRoutesSyncPeriod: 15, }, &v1core.Node{ ObjectMeta: metav1.ObjectMeta{ @@ -1611,15 +1614,16 @@ func Test_routeReflectorConfiguration(t *testing.T) { { "RR client with IPv4 cluster id", &NetworkRoutingController{ - bgpFullMeshMode: false, - bgpPort: 10000, - clientset: fake.NewSimpleClientset(), - nodeIP: net.ParseIP("10.0.0.0"), - routerID: "10.0.0.0", - bgpServer: gobgp.NewBgpServer(), - activeNodes: make(map[string]bool), - nodeAsnNumber: 100, - hostnameOverride: "node-1", + bgpFullMeshMode: false, + bgpPort: 10000, + clientset: fake.NewSimpleClientset(), + nodeIP: net.ParseIP("10.0.0.0"), + routerID: "10.0.0.0", + bgpServer: gobgp.NewBgpServer(), + activeNodes: make(map[string]bool), + nodeAsnNumber: 100, + hostnameOverride: "node-1", + injectedRoutesSyncPeriod: 15, }, &v1core.Node{ ObjectMeta: metav1.ObjectMeta{ @@ -1638,14 +1642,15 @@ func Test_routeReflectorConfiguration(t *testing.T) { { "RR server with unparseable cluster id", &NetworkRoutingController{ - bgpFullMeshMode: false, - bgpPort: 10000, - clientset: fake.NewSimpleClientset(), - nodeIP: net.ParseIP("10.0.0.0"), - bgpServer: gobgp.NewBgpServer(), - activeNodes: make(map[string]bool), - nodeAsnNumber: 100, - hostnameOverride: "node-1", + bgpFullMeshMode: false, + bgpPort: 10000, + clientset: fake.NewSimpleClientset(), + nodeIP: net.ParseIP("10.0.0.0"), + bgpServer: gobgp.NewBgpServer(), + activeNodes: make(map[string]bool), + nodeAsnNumber: 100, + hostnameOverride: "node-1", + injectedRoutesSyncPeriod: 15, }, &v1core.Node{ ObjectMeta: metav1.ObjectMeta{ @@ -1664,14 +1669,15 @@ func Test_routeReflectorConfiguration(t *testing.T) { { "RR client with unparseable cluster id", &NetworkRoutingController{ - bgpFullMeshMode: false, - bgpPort: 10000, - clientset: fake.NewSimpleClientset(), - nodeIP: net.ParseIP("10.0.0.0"), - bgpServer: gobgp.NewBgpServer(), - activeNodes: make(map[string]bool), - nodeAsnNumber: 100, - hostnameOverride: "node-1", + bgpFullMeshMode: false, + bgpPort: 10000, + clientset: fake.NewSimpleClientset(), + nodeIP: net.ParseIP("10.0.0.0"), + bgpServer: gobgp.NewBgpServer(), + activeNodes: make(map[string]bool), + nodeAsnNumber: 100, + hostnameOverride: "node-1", + injectedRoutesSyncPeriod: 15, }, &v1core.Node{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/options/options.go b/pkg/options/options.go index dfc65d75..a6d18d47 100644 --- a/pkg/options/options.go +++ b/pkg/options/options.go @@ -42,6 +42,7 @@ type KubeRouterConfig struct { HealthPort uint16 HelpRequested bool HostnameOverride string + InjectedRoutesSyncPeriod time.Duration IPTablesSyncPeriod time.Duration IpvsGracefulPeriod time.Duration IpvsGracefulTermination bool @@ -89,6 +90,7 @@ func NewKubeRouterConfig() *KubeRouterConfig { NodePortRange: "30000-32767", OverlayType: "subnet", RoutesSyncPeriod: 5 * time.Minute, + InjectedRoutesSyncPeriod: 15 * time.Second, } } @@ -149,6 +151,8 @@ func (s *KubeRouterConfig) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&s.HostnameOverride, "hostname-override", s.HostnameOverride, "Overrides the NodeName of the node. Set this if kube-router is unable to determine your NodeName "+ "automatically.") + fs.DurationVar(&s.InjectedRoutesSyncPeriod, "injected-routes-sync-period", s.InjectedRoutesSyncPeriod, + "The delay between route table synchronizations (e.g. '5s', '1m', '2h22m'). Must be greater than 0.") fs.DurationVar(&s.IPTablesSyncPeriod, "iptables-sync-period", s.IPTablesSyncPeriod, "The delay between iptables rule synchronizations (e.g. '5s', '1m'). Must be greater than 0.") fs.DurationVar(&s.IpvsGracefulPeriod, "ipvs-graceful-period", s.IpvsGracefulPeriod,