diff --git a/pkg/controllers/proxy/network_services_controller.go b/pkg/controllers/proxy/network_services_controller.go index 31c6ba8a..a50eeec8 100644 --- a/pkg/controllers/proxy/network_services_controller.go +++ b/pkg/controllers/proxy/network_services_controller.go @@ -38,18 +38,23 @@ import ( ) const ( - KUBE_DUMMY_IF = "kube-dummy-if" - KUBE_TUNNEL_IF = "kube-tunnel-if" - IFACE_NOT_FOUND = "Link not found" - IFACE_HAS_ADDR = "file exists" - IFACE_HAS_NO_ADDR = "cannot assign requested address" - IPVS_SERVER_EXISTS = "file exists" + KUBE_DUMMY_IF = "kube-dummy-if" + KUBE_TUNNEL_IF = "kube-tunnel-if" + IFACE_NOT_FOUND = "Link not found" + IFACE_HAS_ADDR = "file exists" + IFACE_HAS_NO_ADDR = "cannot assign requested address" + IPVS_SERVER_EXISTS = "file exists" + IPVS_MAGLEV_HASHING = "mh" + IPVS_SVC_F_SCHED1 = "flag-1" + IPVS_SVC_F_SCHED2 = "flag-2" + IPVS_SVC_F_SCHED3 = "flag-3" - svcDSRAnnotation = "kube-router.io/service.dsr" - svcSchedulerAnnotation = "kube-router.io/service.scheduler" - svcHairpinAnnotation = "kube-router.io/service.hairpin" - svcLocalAnnotation = "kube-router.io/service.local" - svcSkipLbIpsAnnotation = "kube-router.io/service.skiplbips" + svcDSRAnnotation = "kube-router.io/service.dsr" + svcSchedulerAnnotation = "kube-router.io/service.scheduler" + svcHairpinAnnotation = "kube-router.io/service.hairpin" + svcLocalAnnotation = "kube-router.io/service.local" + svcSkipLbIpsAnnotation = "kube-router.io/service.skiplbips" + svcSchedFlagsAnnotation = "kube-router.io/service.schedflags" LeaderElectionRecordAnnotationKey = "control-plane.alpha.kubernetes.io/leader" ) @@ -61,7 +66,7 @@ var ( type ipvsCalls interface { ipvsNewService(ipvsSvc *ipvs.Service) error - ipvsAddService(svcs []*ipvs.Service, vip net.IP, protocol, port uint16, persistent bool, scheduler string) (*ipvs.Service, error) + ipvsAddService(svcs []*ipvs.Service, vip net.IP, protocol, port uint16, persistent bool, scheduler string, flags schedFlags) (*ipvs.Service, error) ipvsDelService(ipvsSvc *ipvs.Service) error ipvsUpdateService(ipvsSvc *ipvs.Service) error ipvsGetServices() ([]*ipvs.Service, error) @@ -70,7 +75,7 @@ type ipvsCalls interface { ipvsUpdateDestination(ipvsSvc *ipvs.Service, ipvsDst *ipvs.Destination) error ipvsGetDestinations(ipvsSvc *ipvs.Service) ([]*ipvs.Destination, error) ipvsDelDestination(ipvsSvc *ipvs.Service, ipvsDst *ipvs.Destination) error - ipvsAddFWMarkService(vip net.IP, protocol, port uint16, persistent bool, scheduler string) (*ipvs.Service, error) + ipvsAddFWMarkService(vip net.IP, protocol, port uint16, persistent bool, scheduler string, flags schedFlags) (*ipvs.Service, error) } type netlinkCalls interface { @@ -234,6 +239,14 @@ type serviceInfo struct { externalIPs []string loadBalancerIPs []string local bool + flags schedFlags +} + +// IPVS scheduler flags +type schedFlags struct { + flag1 bool /* ipvs scheduler flag-1 */ + flag2 bool /* ipvs scheduler flag-2 */ + flag3 bool /* ipvs scheduler flag-3 */ } // map of all services, with unique service id(namespace name, service name, port) as key @@ -552,7 +565,7 @@ func (nsc *NetworkServicesController) syncIpvsServices(serviceInfoMap serviceInf } // create IPVS service for the service to be exposed through the cluster ip - ipvsClusterVipSvc, err := nsc.ln.ipvsAddService(ipvsSvcs, svc.clusterIP, protocol, uint16(svc.port), svc.sessionAffinity, svc.scheduler) + ipvsClusterVipSvc, err := nsc.ln.ipvsAddService(ipvsSvcs, svc.clusterIP, protocol, uint16(svc.port), svc.sessionAffinity, svc.scheduler, svc.flags) if err != nil { glog.Errorf("Failed to create ipvs service for cluster ip: %s", err.Error()) continue @@ -584,7 +597,7 @@ func (nsc *NetworkServicesController) syncIpvsServices(serviceInfoMap serviceInf nodeServiceIds = make([]string, len(addrs)) for i, addr := range addrs { - ipvsNodeportSvcs[i], err = nsc.ln.ipvsAddService(ipvsSvcs, addr.IP, protocol, uint16(svc.nodePort), svc.sessionAffinity, svc.scheduler) + ipvsNodeportSvcs[i], err = nsc.ln.ipvsAddService(ipvsSvcs, addr.IP, protocol, uint16(svc.nodePort), svc.sessionAffinity, svc.scheduler, svc.flags) if err != nil { glog.Errorf("Failed to create ipvs service for node port due to: %s", err.Error()) continue @@ -595,7 +608,7 @@ func (nsc *NetworkServicesController) syncIpvsServices(serviceInfoMap serviceInf } } else { ipvsNodeportSvcs = make([]*ipvs.Service, 1) - ipvsNodeportSvcs[0], err = nsc.ln.ipvsAddService(ipvsSvcs, nsc.nodeIP, protocol, uint16(svc.nodePort), svc.sessionAffinity, svc.scheduler) + ipvsNodeportSvcs[0], err = nsc.ln.ipvsAddService(ipvsSvcs, nsc.nodeIP, protocol, uint16(svc.nodePort), svc.sessionAffinity, svc.scheduler, svc.flags) if err != nil { glog.Errorf("Failed to create ipvs service for node port due to: %s", err.Error()) continue @@ -621,7 +634,7 @@ func (nsc *NetworkServicesController) syncIpvsServices(serviceInfoMap serviceInf for _, externalIP := range extIPSet.List() { var externalIpServiceId string if svc.directServerReturn && svc.directServerReturnMethod == "tunnel" { - ipvsExternalIPSvc, err := nsc.ln.ipvsAddFWMarkService(net.ParseIP(externalIP), protocol, uint16(svc.port), svc.sessionAffinity, svc.scheduler) + ipvsExternalIPSvc, err := nsc.ln.ipvsAddFWMarkService(net.ParseIP(externalIP), protocol, uint16(svc.port), svc.sessionAffinity, svc.scheduler, svc.flags) if err != nil { glog.Errorf("Failed to create ipvs service for External IP: %s due to: %s", externalIP, err.Error()) continue @@ -655,7 +668,7 @@ func (nsc *NetworkServicesController) syncIpvsServices(serviceInfoMap serviceInf } // create IPVS service for the service to be exposed through the external ip - ipvsExternalIPSvc, err := nsc.ln.ipvsAddService(ipvsSvcs, net.ParseIP(externalIP), protocol, uint16(svc.port), svc.sessionAffinity, svc.scheduler) + ipvsExternalIPSvc, err := nsc.ln.ipvsAddService(ipvsSvcs, net.ParseIP(externalIP), protocol, uint16(svc.port), svc.sessionAffinity, svc.scheduler, svc.flags) if err != nil { glog.Errorf("Failed to create ipvs service for external ip: %s due to %s", externalIP, err.Error()) continue @@ -1079,8 +1092,16 @@ func (nsc *NetworkServicesController) buildServicesInfo() serviceInfoMap { svcInfo.scheduler = ipvs.DestinationHashing } else if schedulingMethod == ipvs.SourceHashing { svcInfo.scheduler = ipvs.SourceHashing + } else if schedulingMethod == IPVS_MAGLEV_HASHING { + svcInfo.scheduler = IPVS_MAGLEV_HASHING } } + + flags, ok := svc.ObjectMeta.Annotations[svcSchedFlagsAnnotation] + if ok && svcInfo.scheduler == IPVS_MAGLEV_HASHING { + svcInfo.flags = parseSchedFlags(flags) + } + copy(svcInfo.externalIPs, svc.Spec.ExternalIPs) for _, lbIngress := range svc.Status.LoadBalancer.Ingress { if len(lbIngress.IP) > 0 { @@ -1102,6 +1123,32 @@ func (nsc *NetworkServicesController) buildServicesInfo() serviceInfoMap { return serviceMap } +func parseSchedFlags(value string) schedFlags { + var flag1, flag2, flag3 bool + + if len(value) < 1 { + return schedFlags{} + } + + flags := strings.Split(value, ",") + for _, flag := range flags { + switch strings.Trim(flag, " ") { + case IPVS_SVC_F_SCHED1: + flag1 = true + break + case IPVS_SVC_F_SCHED2: + flag2 = true + break + case IPVS_SVC_F_SCHED3: + flag3 = true + break + default: + } + } + + return schedFlags{flag1, flag2, flag3} +} + func shuffle(endPoints []endpointsInfo) []endpointsInfo { for index1 := range endPoints { index2 := rand.Intn(index1 + 1) @@ -1412,6 +1459,18 @@ func ipvsServiceString(s *ipvs.Service) string { flags = flags + "[one-packet scheduling]" } + if s.Flags&0x0008 != 0 { + flags = flags + "[flag-1(fallback)]" + } + + if s.Flags&0x0010 != 0 { + flags = flags + "[flag-2(port)]" + } + + if s.Flags&0x0020 != 0 { + flags = flags + "[flag-3]" + } + return fmt.Sprintf("%s:%s:%v (Flags: %s)", protocol, s.Address, s.Port, flags) } @@ -1432,7 +1491,51 @@ func ipvsSetPersistence(svc *ipvs.Service, p bool) { } } -func (ln *linuxNetworking) ipvsAddService(svcs []*ipvs.Service, vip net.IP, protocol, port uint16, persistent bool, scheduler string) (*ipvs.Service, error) { +func ipvsSetSchedFlags(svc *ipvs.Service, s schedFlags) { + if s.flag1 { + svc.Flags |= 0x0008 + } else { + svc.Flags &^= 0x0008 + } + + if s.flag2 { + svc.Flags |= 0x0010 + } else { + svc.Flags &^= 0x0010 + } + + if s.flag3 { + svc.Flags |= 0x0020 + } else { + svc.Flags &^= 0x0020 + } + + /* Keep netmask which is set by ipvsSetPersistence() before */ + if (svc.Netmask&0xFFFFFFFF != 0) || (s.flag1 || s.flag2 || s.flag3) { + svc.Netmask |= 0xFFFFFFFF + } else { + svc.Netmask &^= 0xFFFFFFFF + } +} + +/* Compare service scheduler flags with ipvs service */ +func changedIpvsSchedFlags(svc *ipvs.Service, s schedFlags) bool { + if (s.flag1 && (svc.Flags&0x0008) == 0) || (!s.flag1 && (svc.Flags&0x0008) != 0) { + return true + } + + if (s.flag2 && (svc.Flags&0x0010) == 0) || (!s.flag2 && (svc.Flags&0x0010) != 0) { + return true + } + + if (s.flag3 && (svc.Flags&0x0020) == 0) || (!s.flag3 && (svc.Flags&0x0020) != 0) { + return true + } + + return false +} + +func (ln *linuxNetworking) ipvsAddService(svcs []*ipvs.Service, vip net.IP, protocol, port uint16, persistent bool, scheduler string, flags schedFlags) (*ipvs.Service, error) { var err error for _, svc := range svcs { @@ -1440,6 +1543,10 @@ func (ln *linuxNetworking) ipvsAddService(svcs []*ipvs.Service, vip net.IP, prot if (persistent && (svc.Flags&0x0001) == 0) || (!persistent && (svc.Flags&0x0001) != 0) { ipvsSetPersistence(svc, persistent) + if changedIpvsSchedFlags(svc, flags) { + ipvsSetSchedFlags(svc, flags) + } + err = ln.ipvsUpdateService(svc) if err != nil { return nil, err @@ -1447,6 +1554,16 @@ func (ln *linuxNetworking) ipvsAddService(svcs []*ipvs.Service, vip net.IP, prot glog.V(2).Infof("Updated persistence/session-affinity for service: %s", ipvsServiceString(svc)) } + if changedIpvsSchedFlags(svc, flags) { + ipvsSetSchedFlags(svc, flags) + + err = ln.ipvsUpdateService(svc) + if err != nil { + return nil, err + } + glog.V(2).Infof("Updated scheduler flags for service: %s", ipvsServiceString(svc)) + } + if scheduler != svc.SchedName { svc.SchedName = scheduler err = ln.ipvsUpdateService(svc) @@ -1472,6 +1589,7 @@ func (ln *linuxNetworking) ipvsAddService(svcs []*ipvs.Service, vip net.IP, prot } ipvsSetPersistence(&svc, persistent) + ipvsSetSchedFlags(&svc, flags) err = ln.ipvsNewService(&svc) if err != nil { @@ -1492,7 +1610,7 @@ func generateFwmark(ip, protocol, port string) uint32 { } // ipvsAddFWMarkService: creates a IPVS service using FWMARK -func (ln *linuxNetworking) ipvsAddFWMarkService(vip net.IP, protocol, port uint16, persistent bool, scheduler string) (*ipvs.Service, error) { +func (ln *linuxNetworking) ipvsAddFWMarkService(vip net.IP, protocol, port uint16, persistent bool, scheduler string, flags schedFlags) (*ipvs.Service, error) { var protocolStr string if protocol == syscall.IPPROTO_TCP { @@ -1516,6 +1634,10 @@ func (ln *linuxNetworking) ipvsAddFWMarkService(vip net.IP, protocol, port uint1 if (persistent && (svc.Flags&0x0001) == 0) || (!persistent && (svc.Flags&0x0001) != 0) { ipvsSetPersistence(svc, persistent) + if changedIpvsSchedFlags(svc, flags) { + ipvsSetSchedFlags(svc, flags) + } + err = ln.ipvsUpdateService(svc) if err != nil { return nil, err @@ -1523,6 +1645,16 @@ func (ln *linuxNetworking) ipvsAddFWMarkService(vip net.IP, protocol, port uint1 glog.V(2).Infof("Updated persistence/session-affinity for service: %s", ipvsServiceString(svc)) } + if changedIpvsSchedFlags(svc, flags) { + ipvsSetSchedFlags(svc, flags) + + err = ln.ipvsUpdateService(svc) + if err != nil { + return nil, err + } + glog.V(2).Infof("Updated scheduler flags for service: %s", ipvsServiceString(svc)) + } + if scheduler != svc.SchedName { svc.SchedName = scheduler err = ln.ipvsUpdateService(svc) @@ -1548,6 +1680,7 @@ func (ln *linuxNetworking) ipvsAddFWMarkService(vip net.IP, protocol, port uint1 } ipvsSetPersistence(&svc, persistent) + ipvsSetSchedFlags(&svc, flags) err = ln.ipvsNewService(&svc) if err != nil { diff --git a/pkg/controllers/proxy/network_services_controller_moq.go b/pkg/controllers/proxy/network_services_controller_moq.go index 89e01630..24ea234a 100644 --- a/pkg/controllers/proxy/network_services_controller_moq.go +++ b/pkg/controllers/proxy/network_services_controller_moq.go @@ -49,13 +49,13 @@ var ( // ipAddrDelFunc: func(iface netlink.Link, ip string) error { // panic("TODO: mock out the ipAddrDel method") // }, -// ipvsAddFWMarkServiceFunc: func(vip net.IP, protocol uint16, port uint16, persistent bool, scheduler string) (*ipvs.Service, error) { +// ipvsAddFWMarkServiceFunc: func(vip net.IP, protocol uint16, port uint16, persistent bool, scheduler string, flags schedFlags) (*ipvs.Service, error) { // panic("TODO: mock out the ipvsAddFWMarkService method") // }, // ipvsAddServerFunc: func(ipvsSvc *ipvs.Service, ipvsDst *ipvs.Destination) error { // panic("TODO: mock out the ipvsAddServer method") // }, -// ipvsAddServiceFunc: func(svcs []*ipvs.Service, vip net.IP, protocol uint16, port uint16, persistent bool, scheduler string) (*ipvs.Service, error) { +// ipvsAddServiceFunc: func(svcs []*ipvs.Service, vip net.IP, protocol uint16, port uint16, persistent bool, scheduler string, flags schedFlags) (*ipvs.Service, error) { // panic("TODO: mock out the ipvsAddService method") // }, // ipvsDelDestinationFunc: func(ipvsSvc *ipvs.Service, ipvsDst *ipvs.Destination) error { @@ -111,13 +111,13 @@ type LinuxNetworkingMock struct { ipAddrDelFunc func(iface netlink.Link, ip string) error // ipvsAddFWMarkServiceFunc mocks the ipvsAddFWMarkService method. - ipvsAddFWMarkServiceFunc func(vip net.IP, protocol uint16, port uint16, persistent bool, scheduler string) (*ipvs.Service, error) + ipvsAddFWMarkServiceFunc func(vip net.IP, protocol uint16, port uint16, persistent bool, scheduler string, flags schedFlags) (*ipvs.Service, error) // ipvsAddServerFunc mocks the ipvsAddServer method. ipvsAddServerFunc func(ipvsSvc *ipvs.Service, ipvsDst *ipvs.Destination) error // ipvsAddServiceFunc mocks the ipvsAddService method. - ipvsAddServiceFunc func(svcs []*ipvs.Service, vip net.IP, protocol uint16, port uint16, persistent bool, scheduler string) (*ipvs.Service, error) + ipvsAddServiceFunc func(svcs []*ipvs.Service, vip net.IP, protocol uint16, port uint16, persistent bool, scheduler string, flags schedFlags) (*ipvs.Service, error) // ipvsDelDestinationFunc mocks the ipvsDelDestination method. ipvsDelDestinationFunc func(ipvsSvc *ipvs.Service, ipvsDst *ipvs.Destination) error @@ -196,6 +196,8 @@ type LinuxNetworkingMock struct { Persistent bool // Scheduler is the scheduler argument value. Scheduler string + // Flags is the flags argument value. + Flags schedFlags } // ipvsAddServer holds details about calls to the ipvsAddServer method. ipvsAddServer []struct { @@ -218,6 +220,8 @@ type LinuxNetworkingMock struct { Persistent bool // Scheduler is the scheduler argument value. Scheduler string + // Flags is the flags argument value. + Flags schedFlags } // ipvsDelDestination holds details about calls to the ipvsDelDestination method. ipvsDelDestination []struct { @@ -427,7 +431,7 @@ func (mock *LinuxNetworkingMock) ipAddrDelCalls() []struct { } // ipvsAddFWMarkService calls ipvsAddFWMarkServiceFunc. -func (mock *LinuxNetworkingMock) ipvsAddFWMarkService(vip net.IP, protocol uint16, port uint16, persistent bool, scheduler string) (*ipvs.Service, error) { +func (mock *LinuxNetworkingMock) ipvsAddFWMarkService(vip net.IP, protocol uint16, port uint16, persistent bool, scheduler string, flags schedFlags) (*ipvs.Service, error) { if mock.ipvsAddFWMarkServiceFunc == nil { panic("LinuxNetworkingMock.ipvsAddFWMarkServiceFunc: method is nil but LinuxNetworking.ipvsAddFWMarkService was just called") } @@ -437,17 +441,19 @@ func (mock *LinuxNetworkingMock) ipvsAddFWMarkService(vip net.IP, protocol uint1 Port uint16 Persistent bool Scheduler string + Flags schedFlags }{ Vip: vip, Protocol: protocol, Port: port, Persistent: persistent, Scheduler: scheduler, + Flags: flags, } lockLinuxNetworkingMockipvsAddFWMarkService.Lock() mock.calls.ipvsAddFWMarkService = append(mock.calls.ipvsAddFWMarkService, callInfo) lockLinuxNetworkingMockipvsAddFWMarkService.Unlock() - return mock.ipvsAddFWMarkServiceFunc(vip, protocol, port, persistent, scheduler) + return mock.ipvsAddFWMarkServiceFunc(vip, protocol, port, persistent, scheduler, flags) } // ipvsAddFWMarkServiceCalls gets all the calls that were made to ipvsAddFWMarkService. @@ -459,6 +465,7 @@ func (mock *LinuxNetworkingMock) ipvsAddFWMarkServiceCalls() []struct { Port uint16 Persistent bool Scheduler string + Flags schedFlags } { var calls []struct { Vip net.IP @@ -466,6 +473,7 @@ func (mock *LinuxNetworkingMock) ipvsAddFWMarkServiceCalls() []struct { Port uint16 Persistent bool Scheduler string + Flags schedFlags } lockLinuxNetworkingMockipvsAddFWMarkService.RLock() calls = mock.calls.ipvsAddFWMarkService @@ -509,7 +517,7 @@ func (mock *LinuxNetworkingMock) ipvsAddServerCalls() []struct { } // ipvsAddService calls ipvsAddServiceFunc. -func (mock *LinuxNetworkingMock) ipvsAddService(svcs []*ipvs.Service, vip net.IP, protocol uint16, port uint16, persistent bool, scheduler string) (*ipvs.Service, error) { +func (mock *LinuxNetworkingMock) ipvsAddService(svcs []*ipvs.Service, vip net.IP, protocol uint16, port uint16, persistent bool, scheduler string, flags schedFlags) (*ipvs.Service, error) { if mock.ipvsAddServiceFunc == nil { panic("LinuxNetworkingMock.ipvsAddServiceFunc: method is nil but LinuxNetworking.ipvsAddService was just called") } @@ -520,6 +528,7 @@ func (mock *LinuxNetworkingMock) ipvsAddService(svcs []*ipvs.Service, vip net.IP Port uint16 Persistent bool Scheduler string + Flags schedFlags }{ Svcs: svcs, Vip: vip, @@ -527,11 +536,12 @@ func (mock *LinuxNetworkingMock) ipvsAddService(svcs []*ipvs.Service, vip net.IP Port: port, Persistent: persistent, Scheduler: scheduler, + Flags: flags, } lockLinuxNetworkingMockipvsAddService.Lock() mock.calls.ipvsAddService = append(mock.calls.ipvsAddService, callInfo) lockLinuxNetworkingMockipvsAddService.Unlock() - return mock.ipvsAddServiceFunc(svcs, vip, protocol, port, persistent, scheduler) + return mock.ipvsAddServiceFunc(svcs, vip, protocol, port, persistent, scheduler, flags) } // ipvsAddServiceCalls gets all the calls that were made to ipvsAddService. @@ -544,6 +554,7 @@ func (mock *LinuxNetworkingMock) ipvsAddServiceCalls() []struct { Port uint16 Persistent bool Scheduler string + Flags schedFlags } { var calls []struct { Svcs []*ipvs.Service @@ -552,6 +563,7 @@ func (mock *LinuxNetworkingMock) ipvsAddServiceCalls() []struct { Port uint16 Persistent bool Scheduler string + Flags schedFlags } lockLinuxNetworkingMockipvsAddService.RLock() calls = mock.calls.ipvsAddService diff --git a/pkg/controllers/proxy/network_services_controller_test.go b/pkg/controllers/proxy/network_services_controller_test.go index 9fd7f71b..6ceb2616 100644 --- a/pkg/controllers/proxy/network_services_controller_test.go +++ b/pkg/controllers/proxy/network_services_controller_test.go @@ -52,7 +52,7 @@ func (lnm *LinuxNetworkingMockImpl) ipAddrAdd(iface netlink.Link, addr string, a func (lnm *LinuxNetworkingMockImpl) ipvsAddServer(ipvsSvc *ipvs.Service, ipvsDst *ipvs.Destination) error { return nil } -func (lnm *LinuxNetworkingMockImpl) ipvsAddService(svcs []*ipvs.Service, vip net.IP, protocol, port uint16, persistent bool, scheduler string) (*ipvs.Service, error) { +func (lnm *LinuxNetworkingMockImpl) ipvsAddService(svcs []*ipvs.Service, vip net.IP, protocol, port uint16, persistent bool, scheduler string, flags schedFlags) (*ipvs.Service, error) { svc := &ipvs.Service{ Address: vip, Protocol: protocol, @@ -179,8 +179,8 @@ var _ = Describe("NetworkServicesController", func() { }) JustBeforeEach(func() { // pre-inject some foo ipvs Service to verify its deletion - fooSvc1, _ = lnm.ipvsAddService(lnm.ipvsSvcs, net.ParseIP("1.2.3.4"), 6, 1234, false, "rr") - fooSvc2, _ = lnm.ipvsAddService(lnm.ipvsSvcs, net.ParseIP("5.6.7.8"), 6, 5678, false, "rr") + fooSvc1, _ = lnm.ipvsAddService(lnm.ipvsSvcs, net.ParseIP("1.2.3.4"), 6, 1234, false, "rr", schedFlags{}) + fooSvc2, _ = lnm.ipvsAddService(lnm.ipvsSvcs, net.ParseIP("5.6.7.8"), 6, 5678, false, "rr", schedFlags{true, true, false}) syncErr = nsc.syncIpvsServices(nsc.serviceMap, nsc.endpointsMap) }) It("Should have called syncIpvsServices OK", func() {