Integrate ip_vs_mh scheduler into kube-router (#564)

* Add to set ip_vs_mh scheduler and flags

Signed-off-by: Inju Song <inju.song@navercorp.com>

* Use scheduler flags when adding or updating service

Signed-off-by: Inju Song <inju.song@navercorp.com>

* Refactor with gofmt, generate moq file and fix test source

Signed-off-by: Inju Song <inju.song@navercorp.com>

pkg/controllers/proxy/network_services_controller.go

@@ -38,18 +38,23 @@ import (
 )
 
 const (
-	KUBE_DUMMY_IF      = "kube-dummy-if"
+	KUBE_DUMMY_IF       = "kube-dummy-if"
-	KUBE_TUNNEL_IF     = "kube-tunnel-if"
+	KUBE_TUNNEL_IF      = "kube-tunnel-if"
-	IFACE_NOT_FOUND    = "Link not found"
+	IFACE_NOT_FOUND     = "Link not found"
-	IFACE_HAS_ADDR     = "file exists"
+	IFACE_HAS_ADDR      = "file exists"
-	IFACE_HAS_NO_ADDR  = "cannot assign requested address"
+	IFACE_HAS_NO_ADDR   = "cannot assign requested address"
-	IPVS_SERVER_EXISTS = "file exists"
+	IPVS_SERVER_EXISTS  = "file exists"
+	IPVS_MAGLEV_HASHING = "mh"
+	IPVS_SVC_F_SCHED1   = "flag-1"
+	IPVS_SVC_F_SCHED2   = "flag-2"
+	IPVS_SVC_F_SCHED3   = "flag-3"
 
-	svcDSRAnnotation       = "kube-router.io/service.dsr"
+	svcDSRAnnotation        = "kube-router.io/service.dsr"
-	svcSchedulerAnnotation = "kube-router.io/service.scheduler"
+	svcSchedulerAnnotation  = "kube-router.io/service.scheduler"
-	svcHairpinAnnotation   = "kube-router.io/service.hairpin"
+	svcHairpinAnnotation    = "kube-router.io/service.hairpin"
-	svcLocalAnnotation     = "kube-router.io/service.local"
+	svcLocalAnnotation      = "kube-router.io/service.local"
-	svcSkipLbIpsAnnotation = "kube-router.io/service.skiplbips"
+	svcSkipLbIpsAnnotation  = "kube-router.io/service.skiplbips"
+	svcSchedFlagsAnnotation = "kube-router.io/service.schedflags"
 
 	LeaderElectionRecordAnnotationKey = "control-plane.alpha.kubernetes.io/leader"
 )
@@ -61,7 +66,7 @@ var (
 
 type ipvsCalls interface {
 	ipvsNewService(ipvsSvc *ipvs.Service) error
-	ipvsAddService(svcs []*ipvs.Service, vip net.IP, protocol, port uint16, persistent bool, scheduler string) (*ipvs.Service, error)
+	ipvsAddService(svcs []*ipvs.Service, vip net.IP, protocol, port uint16, persistent bool, scheduler string, flags schedFlags) (*ipvs.Service, error)
 	ipvsDelService(ipvsSvc *ipvs.Service) error
 	ipvsUpdateService(ipvsSvc *ipvs.Service) error
 	ipvsGetServices() ([]*ipvs.Service, error)
@@ -70,7 +75,7 @@ type ipvsCalls interface {
 	ipvsUpdateDestination(ipvsSvc *ipvs.Service, ipvsDst *ipvs.Destination) error
 	ipvsGetDestinations(ipvsSvc *ipvs.Service) ([]*ipvs.Destination, error)
 	ipvsDelDestination(ipvsSvc *ipvs.Service, ipvsDst *ipvs.Destination) error
-	ipvsAddFWMarkService(vip net.IP, protocol, port uint16, persistent bool, scheduler string) (*ipvs.Service, error)
+	ipvsAddFWMarkService(vip net.IP, protocol, port uint16, persistent bool, scheduler string, flags schedFlags) (*ipvs.Service, error)
 }
 
 type netlinkCalls interface {
@@ -234,6 +239,14 @@ type serviceInfo struct {
 	externalIPs              []string
 	loadBalancerIPs          []string
 	local                    bool
+	flags                    schedFlags
+}
+
+// IPVS scheduler flags
+type schedFlags struct {
+	flag1 bool /* ipvs scheduler flag-1 */
+	flag2 bool /* ipvs scheduler flag-2 */
+	flag3 bool /* ipvs scheduler flag-3 */
 }
 
 // map of all services, with unique service id(namespace name, service name, port) as key
@@ -552,7 +565,7 @@ func (nsc *NetworkServicesController) syncIpvsServices(serviceInfoMap serviceInf
 		}
 
 		// create IPVS service for the service to be exposed through the cluster ip
-		ipvsClusterVipSvc, err := nsc.ln.ipvsAddService(ipvsSvcs, svc.clusterIP, protocol, uint16(svc.port), svc.sessionAffinity, svc.scheduler)
+		ipvsClusterVipSvc, err := nsc.ln.ipvsAddService(ipvsSvcs, svc.clusterIP, protocol, uint16(svc.port), svc.sessionAffinity, svc.scheduler, svc.flags)
 		if err != nil {
 			glog.Errorf("Failed to create ipvs service for cluster ip: %s", err.Error())
 			continue
@@ -584,7 +597,7 @@ func (nsc *NetworkServicesController) syncIpvsServices(serviceInfoMap serviceInf
 				nodeServiceIds = make([]string, len(addrs))
 
 				for i, addr := range addrs {
-					ipvsNodeportSvcs[i], err = nsc.ln.ipvsAddService(ipvsSvcs, addr.IP, protocol, uint16(svc.nodePort), svc.sessionAffinity, svc.scheduler)
+					ipvsNodeportSvcs[i], err = nsc.ln.ipvsAddService(ipvsSvcs, addr.IP, protocol, uint16(svc.nodePort), svc.sessionAffinity, svc.scheduler, svc.flags)
 					if err != nil {
 						glog.Errorf("Failed to create ipvs service for node port due to: %s", err.Error())
 						continue
@@ -595,7 +608,7 @@ func (nsc *NetworkServicesController) syncIpvsServices(serviceInfoMap serviceInf
 				}
 			} else {
 				ipvsNodeportSvcs = make([]*ipvs.Service, 1)
-				ipvsNodeportSvcs[0], err = nsc.ln.ipvsAddService(ipvsSvcs, nsc.nodeIP, protocol, uint16(svc.nodePort), svc.sessionAffinity, svc.scheduler)
+				ipvsNodeportSvcs[0], err = nsc.ln.ipvsAddService(ipvsSvcs, nsc.nodeIP, protocol, uint16(svc.nodePort), svc.sessionAffinity, svc.scheduler, svc.flags)
 				if err != nil {
 					glog.Errorf("Failed to create ipvs service for node port due to: %s", err.Error())
 					continue
@@ -621,7 +634,7 @@ func (nsc *NetworkServicesController) syncIpvsServices(serviceInfoMap serviceInf
 		for _, externalIP := range extIPSet.List() {
 			var externalIpServiceId string
 			if svc.directServerReturn && svc.directServerReturnMethod == "tunnel" {
-				ipvsExternalIPSvc, err := nsc.ln.ipvsAddFWMarkService(net.ParseIP(externalIP), protocol, uint16(svc.port), svc.sessionAffinity, svc.scheduler)
+				ipvsExternalIPSvc, err := nsc.ln.ipvsAddFWMarkService(net.ParseIP(externalIP), protocol, uint16(svc.port), svc.sessionAffinity, svc.scheduler, svc.flags)
 				if err != nil {
 					glog.Errorf("Failed to create ipvs service for External IP: %s due to: %s", externalIP, err.Error())
 					continue
@@ -655,7 +668,7 @@ func (nsc *NetworkServicesController) syncIpvsServices(serviceInfoMap serviceInf
 				}
 
 				// create IPVS service for the service to be exposed through the external ip
-				ipvsExternalIPSvc, err := nsc.ln.ipvsAddService(ipvsSvcs, net.ParseIP(externalIP), protocol, uint16(svc.port), svc.sessionAffinity, svc.scheduler)
+				ipvsExternalIPSvc, err := nsc.ln.ipvsAddService(ipvsSvcs, net.ParseIP(externalIP), protocol, uint16(svc.port), svc.sessionAffinity, svc.scheduler, svc.flags)
 				if err != nil {
 					glog.Errorf("Failed to create ipvs service for external ip: %s due to %s", externalIP, err.Error())
 					continue
@@ -1079,8 +1092,16 @@ func (nsc *NetworkServicesController) buildServicesInfo() serviceInfoMap {
 					svcInfo.scheduler = ipvs.DestinationHashing
 				} else if schedulingMethod == ipvs.SourceHashing {
 					svcInfo.scheduler = ipvs.SourceHashing
+				} else if schedulingMethod == IPVS_MAGLEV_HASHING {
+					svcInfo.scheduler = IPVS_MAGLEV_HASHING
 				}
 			}
+
+			flags, ok := svc.ObjectMeta.Annotations[svcSchedFlagsAnnotation]
+			if ok && svcInfo.scheduler == IPVS_MAGLEV_HASHING {
+				svcInfo.flags = parseSchedFlags(flags)
+			}
+
 			copy(svcInfo.externalIPs, svc.Spec.ExternalIPs)
 			for _, lbIngress := range svc.Status.LoadBalancer.Ingress {
 				if len(lbIngress.IP) > 0 {
@@ -1102,6 +1123,32 @@ func (nsc *NetworkServicesController) buildServicesInfo() serviceInfoMap {
 	return serviceMap
 }
 
+func parseSchedFlags(value string) schedFlags {
+	var flag1, flag2, flag3 bool
+
+	if len(value) < 1 {
+		return schedFlags{}
+	}
+
+	flags := strings.Split(value, ",")
+	for _, flag := range flags {
+		switch strings.Trim(flag, " ") {
+		case IPVS_SVC_F_SCHED1:
+			flag1 = true
+			break
+		case IPVS_SVC_F_SCHED2:
+			flag2 = true
+			break
+		case IPVS_SVC_F_SCHED3:
+			flag3 = true
+			break
+		default:
+		}
+	}
+
+	return schedFlags{flag1, flag2, flag3}
+}
+
 func shuffle(endPoints []endpointsInfo) []endpointsInfo {
 	for index1 := range endPoints {
 		index2 := rand.Intn(index1 + 1)
@@ -1412,6 +1459,18 @@ func ipvsServiceString(s *ipvs.Service) string {
 		flags = flags + "[one-packet scheduling]"
 	}
 
+	if s.Flags&0x0008 != 0 {
+		flags = flags + "[flag-1(fallback)]"
+	}
+
+	if s.Flags&0x0010 != 0 {
+		flags = flags + "[flag-2(port)]"
+	}
+
+	if s.Flags&0x0020 != 0 {
+		flags = flags + "[flag-3]"
+	}
+
 	return fmt.Sprintf("%s:%s:%v (Flags: %s)", protocol, s.Address, s.Port, flags)
 }
 
@@ -1432,7 +1491,51 @@ func ipvsSetPersistence(svc *ipvs.Service, p bool) {
 	}
 }
 
-func (ln *linuxNetworking) ipvsAddService(svcs []*ipvs.Service, vip net.IP, protocol, port uint16, persistent bool, scheduler string) (*ipvs.Service, error) {
+func ipvsSetSchedFlags(svc *ipvs.Service, s schedFlags) {
+	if s.flag1 {
+		svc.Flags |= 0x0008
+	} else {
+		svc.Flags &^= 0x0008
+	}
+
+	if s.flag2 {
+		svc.Flags |= 0x0010
+	} else {
+		svc.Flags &^= 0x0010
+	}
+
+	if s.flag3 {
+		svc.Flags |= 0x0020
+	} else {
+		svc.Flags &^= 0x0020
+	}
+
+	/* Keep netmask which is set by ipvsSetPersistence() before */
+	if (svc.Netmask&0xFFFFFFFF != 0) || (s.flag1 || s.flag2 || s.flag3) {
+		svc.Netmask |= 0xFFFFFFFF
+	} else {
+		svc.Netmask &^= 0xFFFFFFFF
+	}
+}
+
+/* Compare service scheduler flags with ipvs service */
+func changedIpvsSchedFlags(svc *ipvs.Service, s schedFlags) bool {
+	if (s.flag1 && (svc.Flags&0x0008) == 0) || (!s.flag1 && (svc.Flags&0x0008) != 0) {
+		return true
+	}
+
+	if (s.flag2 && (svc.Flags&0x0010) == 0) || (!s.flag2 && (svc.Flags&0x0010) != 0) {
+		return true
+	}
+
+	if (s.flag3 && (svc.Flags&0x0020) == 0) || (!s.flag3 && (svc.Flags&0x0020) != 0) {
+		return true
+	}
+
+	return false
+}
+
+func (ln *linuxNetworking) ipvsAddService(svcs []*ipvs.Service, vip net.IP, protocol, port uint16, persistent bool, scheduler string, flags schedFlags) (*ipvs.Service, error) {
 
 	var err error
 	for _, svc := range svcs {
@@ -1440,6 +1543,10 @@ func (ln *linuxNetworking) ipvsAddService(svcs []*ipvs.Service, vip net.IP, prot
 			if (persistent && (svc.Flags&0x0001) == 0) || (!persistent && (svc.Flags&0x0001) != 0) {
 				ipvsSetPersistence(svc, persistent)
 
+				if changedIpvsSchedFlags(svc, flags) {
+					ipvsSetSchedFlags(svc, flags)
+				}
+
 				err = ln.ipvsUpdateService(svc)
 				if err != nil {
 					return nil, err
@@ -1447,6 +1554,16 @@ func (ln *linuxNetworking) ipvsAddService(svcs []*ipvs.Service, vip net.IP, prot
 				glog.V(2).Infof("Updated persistence/session-affinity for service: %s", ipvsServiceString(svc))
 			}
 
+			if changedIpvsSchedFlags(svc, flags) {
+				ipvsSetSchedFlags(svc, flags)
+
+				err = ln.ipvsUpdateService(svc)
+				if err != nil {
+					return nil, err
+				}
+				glog.V(2).Infof("Updated scheduler flags for service: %s", ipvsServiceString(svc))
+			}
+
 			if scheduler != svc.SchedName {
 				svc.SchedName = scheduler
 				err = ln.ipvsUpdateService(svc)
@@ -1472,6 +1589,7 @@ func (ln *linuxNetworking) ipvsAddService(svcs []*ipvs.Service, vip net.IP, prot
 	}
 
 	ipvsSetPersistence(&svc, persistent)
+	ipvsSetSchedFlags(&svc, flags)
 
 	err = ln.ipvsNewService(&svc)
 	if err != nil {
@@ -1492,7 +1610,7 @@ func generateFwmark(ip, protocol, port string) uint32 {
 }
 
 // ipvsAddFWMarkService: creates a IPVS service using FWMARK
-func (ln *linuxNetworking) ipvsAddFWMarkService(vip net.IP, protocol, port uint16, persistent bool, scheduler string) (*ipvs.Service, error) {
+func (ln *linuxNetworking) ipvsAddFWMarkService(vip net.IP, protocol, port uint16, persistent bool, scheduler string, flags schedFlags) (*ipvs.Service, error) {
 
 	var protocolStr string
 	if protocol == syscall.IPPROTO_TCP {
@@ -1516,6 +1634,10 @@ func (ln *linuxNetworking) ipvsAddFWMarkService(vip net.IP, protocol, port uint1
 			if (persistent && (svc.Flags&0x0001) == 0) || (!persistent && (svc.Flags&0x0001) != 0) {
 				ipvsSetPersistence(svc, persistent)
 
+				if changedIpvsSchedFlags(svc, flags) {
+					ipvsSetSchedFlags(svc, flags)
+				}
+
 				err = ln.ipvsUpdateService(svc)
 				if err != nil {
 					return nil, err
@@ -1523,6 +1645,16 @@ func (ln *linuxNetworking) ipvsAddFWMarkService(vip net.IP, protocol, port uint1
 				glog.V(2).Infof("Updated persistence/session-affinity for service: %s", ipvsServiceString(svc))
 			}
 
+			if changedIpvsSchedFlags(svc, flags) {
+				ipvsSetSchedFlags(svc, flags)
+
+				err = ln.ipvsUpdateService(svc)
+				if err != nil {
+					return nil, err
+				}
+				glog.V(2).Infof("Updated scheduler flags for service: %s", ipvsServiceString(svc))
+			}
+
 			if scheduler != svc.SchedName {
 				svc.SchedName = scheduler
 				err = ln.ipvsUpdateService(svc)
@@ -1548,6 +1680,7 @@ func (ln *linuxNetworking) ipvsAddFWMarkService(vip net.IP, protocol, port uint1
 	}
 
 	ipvsSetPersistence(&svc, persistent)
+	ipvsSetSchedFlags(&svc, flags)
 
 	err = ln.ipvsNewService(&svc)
 	if err != nil {

pkg/controllers/proxy/network_services_controller_moq.go

@@ -49,13 +49,13 @@ var (
 //             ipAddrDelFunc: func(iface netlink.Link, ip string) error {
 // 	               panic("TODO: mock out the ipAddrDel method")
 //             },
-//             ipvsAddFWMarkServiceFunc: func(vip net.IP, protocol uint16, port uint16, persistent bool, scheduler string) (*ipvs.Service, error) {
+//             ipvsAddFWMarkServiceFunc: func(vip net.IP, protocol uint16, port uint16, persistent bool, scheduler string, flags schedFlags) (*ipvs.Service, error) {
 // 	               panic("TODO: mock out the ipvsAddFWMarkService method")
 //             },
 //             ipvsAddServerFunc: func(ipvsSvc *ipvs.Service, ipvsDst *ipvs.Destination) error {
 // 	               panic("TODO: mock out the ipvsAddServer method")
 //             },
-//             ipvsAddServiceFunc: func(svcs []*ipvs.Service, vip net.IP, protocol uint16, port uint16, persistent bool, scheduler string) (*ipvs.Service, error) {
+//             ipvsAddServiceFunc: func(svcs []*ipvs.Service, vip net.IP, protocol uint16, port uint16, persistent bool, scheduler string, flags schedFlags) (*ipvs.Service, error) {
 // 	               panic("TODO: mock out the ipvsAddService method")
 //             },
 //             ipvsDelDestinationFunc: func(ipvsSvc *ipvs.Service, ipvsDst *ipvs.Destination) error {
@@ -111,13 +111,13 @@ type LinuxNetworkingMock struct {
 	ipAddrDelFunc func(iface netlink.Link, ip string) error
 
 	// ipvsAddFWMarkServiceFunc mocks the ipvsAddFWMarkService method.
-	ipvsAddFWMarkServiceFunc func(vip net.IP, protocol uint16, port uint16, persistent bool, scheduler string) (*ipvs.Service, error)
+	ipvsAddFWMarkServiceFunc func(vip net.IP, protocol uint16, port uint16, persistent bool, scheduler string, flags schedFlags) (*ipvs.Service, error)
 
 	// ipvsAddServerFunc mocks the ipvsAddServer method.
 	ipvsAddServerFunc func(ipvsSvc *ipvs.Service, ipvsDst *ipvs.Destination) error
 
 	// ipvsAddServiceFunc mocks the ipvsAddService method.
-	ipvsAddServiceFunc func(svcs []*ipvs.Service, vip net.IP, protocol uint16, port uint16, persistent bool, scheduler string) (*ipvs.Service, error)
+	ipvsAddServiceFunc func(svcs []*ipvs.Service, vip net.IP, protocol uint16, port uint16, persistent bool, scheduler string, flags schedFlags) (*ipvs.Service, error)
 
 	// ipvsDelDestinationFunc mocks the ipvsDelDestination method.
 	ipvsDelDestinationFunc func(ipvsSvc *ipvs.Service, ipvsDst *ipvs.Destination) error
@@ -196,6 +196,8 @@ type LinuxNetworkingMock struct {
 			Persistent bool
 			// Scheduler is the scheduler argument value.
 			Scheduler string
+			// Flags is the flags argument value.
+			Flags schedFlags
 		}
 		// ipvsAddServer holds details about calls to the ipvsAddServer method.
 		ipvsAddServer []struct {
@@ -218,6 +220,8 @@ type LinuxNetworkingMock struct {
 			Persistent bool
 			// Scheduler is the scheduler argument value.
 			Scheduler string
+			// Flags is the flags argument value.
+			Flags schedFlags
 		}
 		// ipvsDelDestination holds details about calls to the ipvsDelDestination method.
 		ipvsDelDestination []struct {
@@ -427,7 +431,7 @@ func (mock *LinuxNetworkingMock) ipAddrDelCalls() []struct {
 }
 
 // ipvsAddFWMarkService calls ipvsAddFWMarkServiceFunc.
-func (mock *LinuxNetworkingMock) ipvsAddFWMarkService(vip net.IP, protocol uint16, port uint16, persistent bool, scheduler string) (*ipvs.Service, error) {
+func (mock *LinuxNetworkingMock) ipvsAddFWMarkService(vip net.IP, protocol uint16, port uint16, persistent bool, scheduler string, flags schedFlags) (*ipvs.Service, error) {
 	if mock.ipvsAddFWMarkServiceFunc == nil {
 		panic("LinuxNetworkingMock.ipvsAddFWMarkServiceFunc: method is nil but LinuxNetworking.ipvsAddFWMarkService was just called")
 	}
@@ -437,17 +441,19 @@ func (mock *LinuxNetworkingMock) ipvsAddFWMarkService(vip net.IP, protocol uint1
 		Port       uint16
 		Persistent bool
 		Scheduler  string
+		Flags      schedFlags
 	}{
 		Vip:        vip,
 		Protocol:   protocol,
 		Port:       port,
 		Persistent: persistent,
 		Scheduler:  scheduler,
+		Flags:      flags,
 	}
 	lockLinuxNetworkingMockipvsAddFWMarkService.Lock()
 	mock.calls.ipvsAddFWMarkService = append(mock.calls.ipvsAddFWMarkService, callInfo)
 	lockLinuxNetworkingMockipvsAddFWMarkService.Unlock()
-	return mock.ipvsAddFWMarkServiceFunc(vip, protocol, port, persistent, scheduler)
+	return mock.ipvsAddFWMarkServiceFunc(vip, protocol, port, persistent, scheduler, flags)
 }
 
 // ipvsAddFWMarkServiceCalls gets all the calls that were made to ipvsAddFWMarkService.
@@ -459,6 +465,7 @@ func (mock *LinuxNetworkingMock) ipvsAddFWMarkServiceCalls() []struct {
 	Port       uint16
 	Persistent bool
 	Scheduler  string
+	Flags      schedFlags
 } {
 	var calls []struct {
 		Vip        net.IP
@@ -466,6 +473,7 @@ func (mock *LinuxNetworkingMock) ipvsAddFWMarkServiceCalls() []struct {
 		Port       uint16
 		Persistent bool
 		Scheduler  string
+		Flags      schedFlags
 	}
 	lockLinuxNetworkingMockipvsAddFWMarkService.RLock()
 	calls = mock.calls.ipvsAddFWMarkService
@@ -509,7 +517,7 @@ func (mock *LinuxNetworkingMock) ipvsAddServerCalls() []struct {
 }
 
 // ipvsAddService calls ipvsAddServiceFunc.
-func (mock *LinuxNetworkingMock) ipvsAddService(svcs []*ipvs.Service, vip net.IP, protocol uint16, port uint16, persistent bool, scheduler string) (*ipvs.Service, error) {
+func (mock *LinuxNetworkingMock) ipvsAddService(svcs []*ipvs.Service, vip net.IP, protocol uint16, port uint16, persistent bool, scheduler string, flags schedFlags) (*ipvs.Service, error) {
 	if mock.ipvsAddServiceFunc == nil {
 		panic("LinuxNetworkingMock.ipvsAddServiceFunc: method is nil but LinuxNetworking.ipvsAddService was just called")
 	}
@@ -520,6 +528,7 @@ func (mock *LinuxNetworkingMock) ipvsAddService(svcs []*ipvs.Service, vip net.IP
 		Port       uint16
 		Persistent bool
 		Scheduler  string
+		Flags      schedFlags
 	}{
 		Svcs:       svcs,
 		Vip:        vip,
@@ -527,11 +536,12 @@ func (mock *LinuxNetworkingMock) ipvsAddService(svcs []*ipvs.Service, vip net.IP
 		Port:       port,
 		Persistent: persistent,
 		Scheduler:  scheduler,
+		Flags:      flags,
 	}
 	lockLinuxNetworkingMockipvsAddService.Lock()
 	mock.calls.ipvsAddService = append(mock.calls.ipvsAddService, callInfo)
 	lockLinuxNetworkingMockipvsAddService.Unlock()
-	return mock.ipvsAddServiceFunc(svcs, vip, protocol, port, persistent, scheduler)
+	return mock.ipvsAddServiceFunc(svcs, vip, protocol, port, persistent, scheduler, flags)
 }
 
 // ipvsAddServiceCalls gets all the calls that were made to ipvsAddService.
@@ -544,6 +554,7 @@ func (mock *LinuxNetworkingMock) ipvsAddServiceCalls() []struct {
 	Port       uint16
 	Persistent bool
 	Scheduler  string
+	Flags      schedFlags
 } {
 	var calls []struct {
 		Svcs       []*ipvs.Service
@@ -552,6 +563,7 @@ func (mock *LinuxNetworkingMock) ipvsAddServiceCalls() []struct {
 		Port       uint16
 		Persistent bool
 		Scheduler  string
+		Flags      schedFlags
 	}
 	lockLinuxNetworkingMockipvsAddService.RLock()
 	calls = mock.calls.ipvsAddService

pkg/controllers/proxy/network_services_controller_test.go

@@ -52,7 +52,7 @@ func (lnm *LinuxNetworkingMockImpl) ipAddrAdd(iface netlink.Link, addr string, a
 func (lnm *LinuxNetworkingMockImpl) ipvsAddServer(ipvsSvc *ipvs.Service, ipvsDst *ipvs.Destination) error {
 	return nil
 }
-func (lnm *LinuxNetworkingMockImpl) ipvsAddService(svcs []*ipvs.Service, vip net.IP, protocol, port uint16, persistent bool, scheduler string) (*ipvs.Service, error) {
+func (lnm *LinuxNetworkingMockImpl) ipvsAddService(svcs []*ipvs.Service, vip net.IP, protocol, port uint16, persistent bool, scheduler string, flags schedFlags) (*ipvs.Service, error) {
 	svc := &ipvs.Service{
 		Address:  vip,
 		Protocol: protocol,
@@ -179,8 +179,8 @@ var _ = Describe("NetworkServicesController", func() {
 		})
 		JustBeforeEach(func() {
 			// pre-inject some foo ipvs Service to verify its deletion
-			fooSvc1, _ = lnm.ipvsAddService(lnm.ipvsSvcs, net.ParseIP("1.2.3.4"), 6, 1234, false, "rr")
+			fooSvc1, _ = lnm.ipvsAddService(lnm.ipvsSvcs, net.ParseIP("1.2.3.4"), 6, 1234, false, "rr", schedFlags{})
-			fooSvc2, _ = lnm.ipvsAddService(lnm.ipvsSvcs, net.ParseIP("5.6.7.8"), 6, 5678, false, "rr")
+			fooSvc2, _ = lnm.ipvsAddService(lnm.ipvsSvcs, net.ParseIP("5.6.7.8"), 6, 5678, false, "rr", schedFlags{true, true, false})
 			syncErr = nsc.syncIpvsServices(nsc.serviceMap, nsc.endpointsMap)
 		})
 		It("Should have called syncIpvsServices OK", func() {