mirrors/kube-router
1
0
Fork 0
mirror of https://github.com/cloudnativelabs/kube-router.git synced 2025-10-07 16:01:08 +02:00
Code Issues Packages Projects Releases Wiki Activity

Use service annotations to choose IPVS scheduling method (#207)

Browse Source 
Fixes #6
This commit is contained in:
Murali Reddy 2017-10-25 11:38:50 +05:30 committed by GitHub
parent 6d432681de
commit 77f45e356b
2 changed files with 66 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
Documentation/README.md
View File

@ -216,11 +216,18 @@ kubectl annotate service my-service "kube-router.io/service.hairpin="
### Direct server return
You can enable DSR(Direct Server Return) functionality per service. When enabled service endpoint
will directly respond to the client. When DSR is enabled Kube-router will uses LVS's tunneling mode to achive this.
will directly respond to the client by passign the service proxy. When DSR is enabled Kube-router
will uses LVS's tunneling mode to achieve this.
To enable DSR you need to annotate service with `kube-router.io/service.dsr=tunnel` annotation.
To enable DSR you need to annotate service with `kube-router.io/service.dsr=tunnel` annotation. For e.g.
In the current implementation althouh annotation is enabled, DSR will be applicable only to the external IP's.
```
kubectl annotate service my-service "kube-router.io/service.dsr=tunnel"
```
**In the current implementation when annotation is applied on the service, DSR will be applicable only to the external IP's.**
**Also when DSR is used, current implementation does not support port remapping. So you need to use same port and target port for the service**
You will need to enable `hostIPC: true` and `hostPID: true` in kube-router daemonset manifest.
Also host path `/var/run/docker.sock` must be made a volumemount to kube-router.
@ -228,7 +235,26 @@ Also host path `/var/run/docker.sock` must be made a volumemount to kube-router.
Above changes are required for kube-router to enter pod namespeace and create ipip tunnel in the pod and to
assign the external IP to the VIP.
For an e.g manifest please look at [manifest](../daemonset/kubeadm-kuberouter-all-features-dsr.yaml) with DSR requirments enabled.
For an e.g manifest please look at [manifest](../daemonset/kubeadm-kuberouter-all-features-dsr.yaml) with DSR requirements enabled.
### Load balancing Scheduling Algorithms
Kube-router uses LVS for service proxy. LVS support rich set of [scheduling alogirthms](http://kb.linuxvirtualserver.org/wiki/IPVS#Job_Scheduling_Algorithms). You can annotate
the service to choose one of the scheduling alogirthms. When a service is not annotated `round-robin` scheduler is selected by default
```
For least connection scheduling use:
kubectl annotate service my-service "kube-router.io/service.scheduler=lc"
For round-robin scheduling use:
kubectl annotate service my-service "kube-router.io/service.scheduler=rr"
For source hashing scheduling use:
kubectl annotate service my-service "kube-router.io/service.scheduler=sh"
For destination hashing scheduling use:
kubectl annotate service my-service "kube-router.io/service.scheduler=dh"
```
## BGP configuration

42
app/controllers/network_services_controller.go
View File

@ -97,6 +97,7 @@ type serviceInfo struct {
nodePort int
sessionAffinity bool
directServerReturn bool
scheduler string
directServerReturnMethod string
hairpin bool
externalIPs []string
@ -272,7 +273,7 @@ func (nsc *NetworkServicesController) syncIpvsServices(serviceInfoMap serviceInf
}
// create IPVS service for the service to be exposed through the cluster ip
ipvsClusterVipSvc, err := ipvsAddService(svc.clusterIP, protocol, uint16(svc.port), svc.sessionAffinity)
ipvsClusterVipSvc, err := ipvsAddService(svc.clusterIP, protocol, uint16(svc.port), svc.sessionAffinity, svc.scheduler)
if err != nil {
glog.Errorf("Failed to create ipvs service for cluster ip: %s", err.Error())
continue
@ -288,7 +289,7 @@ func (nsc *NetworkServicesController) syncIpvsServices(serviceInfoMap serviceInf
if vip = nsc.nodeIP; nsc.nodeportBindOnAllIp {
vip = net.ParseIP("127.0.0.1")
}
ipvsNodeportSvc, err = ipvsAddService(vip, protocol, uint16(svc.nodePort), svc.sessionAffinity)
ipvsNodeportSvc, err = ipvsAddService(vip, protocol, uint16(svc.nodePort), svc.sessionAffinity, svc.scheduler)
if err != nil {
glog.Errorf("Failed to create ipvs service for node port due to: %s", err.Error())
continue
@ -310,7 +311,7 @@ func (nsc *NetworkServicesController) syncIpvsServices(serviceInfoMap serviceInf
// without a VIP http://www.austintek.com/LVS/LVS-HOWTO/HOWTO/LVS-HOWTO.routing_to_VIP-less_director.html
// to avoid martian packets
for _, externalIP := range svc.externalIPs {
ipvsExternalIPSvc, err := ipvsAddFWMarkService(net.ParseIP(externalIP), protocol, uint16(svc.port), svc.sessionAffinity)
ipvsExternalIPSvc, err := ipvsAddFWMarkService(net.ParseIP(externalIP), protocol, uint16(svc.port), svc.sessionAffinity, svc.scheduler)
if err != nil {
glog.Errorf("Failed to create ipvs service for External IP: %s due to: %s", externalIP, err.Error())
continue
@ -612,6 +613,19 @@ func buildServicesInfo() serviceInfoMap {
svcInfo.directServerReturn = true
svcInfo.directServerReturnMethod = dsrMethod
}
svcInfo.scheduler = ipvs.RoundRobin
schedulingMethod, ok := svc.ObjectMeta.Annotations["kube-router.io/service.scheduler"]
if ok {
if schedulingMethod == ipvs.RoundRobin {
svcInfo.scheduler = ipvs.RoundRobin
} else if schedulingMethod == ipvs.LeastConnection {
svcInfo.scheduler = ipvs.LeastConnection
} else if schedulingMethod == ipvs.DestinationHashing {
svcInfo.scheduler = ipvs.DestinationHashing
} else if schedulingMethod == ipvs.SourceHashing {
svcInfo.scheduler = ipvs.SourceHashing
}
}
copy(svcInfo.externalIPs, svc.Spec.ExternalIPs)
svcInfo.sessionAffinity = (svc.Spec.SessionAffinity == "ClientIP")
_, svcInfo.hairpin = svc.ObjectMeta.Annotations["kube-router.io/service.hairpin"]
@ -941,7 +955,7 @@ func ipvsSetPersistence(svc *ipvs.Service, p bool) {
}
}
func ipvsAddService(vip net.IP, protocol, port uint16, persistent bool) (*ipvs.Service, error) {
func ipvsAddService(vip net.IP, protocol, port uint16, persistent bool, scheduler string) (*ipvs.Service, error) {
svcs, err := h.GetServices()
if err != nil {
return nil, err
@ -959,6 +973,14 @@ func ipvsAddService(vip net.IP, protocol, port uint16, persistent bool) (*ipvs.S
glog.Infof("Updated persistence/session-affinity for service: %s", ipvsServiceString(svc))
}
if scheduler != svc.SchedName {
svc.SchedName = scheduler
err = h.UpdateService(svc)
if err != nil {
return nil, errors.New("Failed to update the scheduler for the service due to " + err.Error())
}
glog.Infof("Updated schedule for the service: %s", ipvsServiceString(svc))
}
// TODO: Make this debug output when we get log levels
// glog.Fatal("ipvs service %s:%s:%s already exists so returning", vip.String(),
// protocol, strconv.Itoa(int(port)))
@ -972,7 +994,7 @@ func ipvsAddService(vip net.IP, protocol, port uint16, persistent bool) (*ipvs.S
AddressFamily: syscall.AF_INET,
Protocol: protocol,
Port: port,
SchedName: ipvs.RoundRobin,
SchedName: scheduler,
}
ipvsSetPersistence(&svc, persistent)
@ -996,7 +1018,7 @@ func generateFwmark(ip, protocol, port string) uint32 {
}
// ipvsAddFWMarkService: creates a IPVS service using FWMARK
func ipvsAddFWMarkService(vip net.IP, protocol, port uint16, persistent bool) (*ipvs.Service, error) {
func ipvsAddFWMarkService(vip net.IP, protocol, port uint16, persistent bool, scheduler string) (*ipvs.Service, error) {
var protocolStr string
if protocol == syscall.IPPROTO_TCP {
@ -1027,6 +1049,14 @@ func ipvsAddFWMarkService(vip net.IP, protocol, port uint16, persistent bool) (*
glog.Infof("Updated persistence/session-affinity for service: %s", ipvsServiceString(svc))
}
if scheduler != svc.SchedName {
svc.SchedName = scheduler
err = h.UpdateService(svc)
if err != nil {
return nil, errors.New("Failed to update the scheduler for the service due to " + err.Error())
}
glog.Infof("Updated schedule for the service: %s", ipvsServiceString(svc))
}
// TODO: Make this debug output when we get log levels
// glog.Fatal("ipvs service %s:%s:%s already exists so returning", vip.String(),
// protocol, strconv.Itoa(int(port)))