implement per-service annotations to control IP advertisment (#575)

* add unit tests for implementing #75

Signed-off-by: Steven Armstrong <steven.armstrong@id.ethz.ch>

* integration tests for #75

Signed-off-by: Steven Armstrong <steven.armstrong@id.ethz.ch>

* update docs for #75

Signed-off-by: Steven Armstrong <steven.armstrong@id.ethz.ch>

* define new kube-router.io/service.advertise.* annotations

Signed-off-by: Steven Armstrong <steven.armstrong@id.ethz.ch>

* Implement per service annotations for advertising IPs.

Signed-off-by: Steven Armstrong <steven.armstrong@id.ethz.ch>

* more consistent annotation names

Signed-off-by: Steven Armstrong <steven.armstrong@id.ethz.ch>

* remove redundant tests

Signed-off-by: Steven Armstrong <steven.armstrong@id.ethz.ch>

docs/user-guide.md

@@ -134,6 +134,39 @@ and if you want to move back to kube-proxy then clean up config done by kube-rou
 and run kube-proxy with the configuration you have.
 - [General Setup](/README.md#getting-started)
 
+
+## Advertising IPs
+
+kube-router can advertise Cluster, External and LoadBalancer IPs to BGP peers.
+It does this by:
+* locally adding the advertised IPs to the nodes' `kube-dummy-if` network interface
+* advertising the IPs to its BGP peers
+
+To set the default for all services use the `--advertise-cluster-ip`,
+`--advertise-external-ip` and `--advertise-loadbalancer-ip` flags.
+
+To selectively enable or disable this feature per-service use the
+`kube-router.io/service.advertise.cluster`, `kube-router.io/service.advertise.external`
+and `kube-router.io/service.advertise.loadbalancer` annotations.
+
+e.g.:
+`$ kubectl annotate service my-advertised-service "kube-router.io/service.advertise.cluster=true"`
+`$ kubectl annotate service my-advertised-service "kube-router.io/service.advertise.external=true"`
+`$ kubectl annotate service my-advertised-service "kube-router.io/service.advertise.loadbalancer=true"`
+
+`$ kubectl annotate service my-non-advertised-service "kube-router.io/service.advertise.cluster=false"`
+`$ kubectl annotate service my-non-advertised-service "kube-router.io/service.advertise.external=false"`
+`$ kubectl annotate service my-non-advertised-service "kube-router.io/service.advertise.loadbalancer=false"`
+
+By combining the flags with the per-service annotations you can choose either
+a opt-in or opt-out strategy for advertising IPs.
+
+Advertising LoadBalancer IPs works by inspecting the services
+`status.loadBalancer.ingress` IPs that are set by external LoadBalancers like
+for example MetalLb. This has been successfully tested together with
+[MetalLB](https://github.com/google/metallb) in ARP mode.
+
+
 ## Hairpin Mode
 
 Communication from a Pod that is behind a Service to its own ClusterIP:Port is
@@ -216,25 +249,6 @@ For destination hashing scheduling use:
 kubectl annotate service my-service "kube-router.io/service.scheduler=dh"
 ```
 
-## LoadBalancer IPs
-
-If you want to also advertise loadbalancer set IPs
-(`status.loadBalancer.ingress` IPs), e.g. when using it with MetalLb,
-add the `--advertise-loadbalancer-ip` flag (`false` by default).
-
-To selectively disable this behaviour per-service, you can use
-the `kube-router.io/service.skiplbips` annotation as e.g.:
-`$ kubectl annotate service my-external-service "kube-router.io/service.skiplbips=true"`
-
-In concrete, unless the Service is annotated as per above, the
-`--advertise-loadbalancer-ip` flag will make Service's Ingress IP(s)
-set by the LoadBalancer to:
-* be locally added to nodes' `kube-dummy-if` network interface
-* be advertised to BGP peers
-
-FYI Above has been successfully tested together with
-[MetalLB](https://github.com/google/metallb) in ARP mode.
-
 ## HostPort support
 
 If you would like to use `HostPort` functionality below changes are required in the manifest.

pkg/controllers/routing/ecmp_vip.go

@@ -268,12 +268,9 @@ func (nrc *NetworkRoutingController) getLoadBalancerIps(svc *v1core.Service) []s
 	if svc.Spec.Type == "LoadBalancer" {
 		// skip headless services
 		if svc.Spec.ClusterIP != "None" && svc.Spec.ClusterIP != "" {
-			_, skiplbips := svc.ObjectMeta.Annotations["kube-router.io/service.skiplbips"]
+			for _, lbIngress := range svc.Status.LoadBalancer.Ingress {
-			if !skiplbips {
+				if len(lbIngress.IP) > 0 {
-				for _, lbIngress := range svc.Status.LoadBalancer.Ingress {
+					loadBalancerIpList = append(loadBalancerIpList, lbIngress.IP)
-					if len(lbIngress.IP) > 0 {
-						loadBalancerIpList = append(loadBalancerIpList, lbIngress.IP)
-					}
 				}
 			}
 		}
@@ -313,6 +310,16 @@ func (nrc *NetworkRoutingController) getVIPs(onlyActiveEndpoints bool) ([]string
 	return toAdvertiseList, toWithdrawList, nil
 }
 
+func (nrc *NetworkRoutingController) shouldAdvertiseService(svc *v1core.Service, annotation string, defaultValue bool) bool {
+	returnValue := defaultValue
+	stringValue, exists := svc.Annotations[annotation]
+	if exists {
+		// Service annotations overrides defaults.
+		returnValue, _ = strconv.ParseBool(stringValue)
+	}
+	return returnValue
+}
+
 func (nrc *NetworkRoutingController) getVIPsForService(svc *v1core.Service, onlyActiveEndpoints bool) ([]string, []string, error) {
 	ipList := make([]string, 0)
 	var err error
@@ -328,16 +335,21 @@ func (nrc *NetworkRoutingController) getVIPsForService(svc *v1core.Service, only
 		}
 	}
 
-	if nrc.advertiseClusterIP {
+	if nrc.shouldAdvertiseService(svc, svcAdvertiseClusterAnnotation, nrc.advertiseClusterIP) {
 		clusterIp := nrc.getClusterIp(svc)
 		if clusterIp != "" {
 			ipList = append(ipList, clusterIp)
 		}
 	}
-	if nrc.advertiseExternalIP {
+
+	if nrc.shouldAdvertiseService(svc, svcAdvertiseExternalAnnotation, nrc.advertiseExternalIP) {
 		ipList = append(ipList, nrc.getExternalIps(svc)...)
 	}
-	if nrc.advertiseLoadBalancerIP {
+
+	// Deprecated: Use service.advertise.loadbalancer=false instead of service.skiplbips.
+	_, skiplbips := svc.Annotations[svcSkipLbIpsAnnotation]
+	advertiseLoadBalancer := nrc.shouldAdvertiseService(svc, svcAdvertiseLoadBalancerAnnotation, nrc.advertiseLoadBalancerIP)
+	if advertiseLoadBalancer && !skiplbips {
 		ipList = append(ipList, nrc.getLoadBalancerIps(svc)...)
 	}
 

pkg/controllers/routing/ecmp_vip_test.go

@@ -0,0 +1,342 @@
+package routing
+
+import (
+	"testing"
+
+	v1core "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/fake"
+)
+
+// Compare 2 string slices by value.
+func Equal(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i, v := range a {
+		if v != b[i] {
+			return false
+		}
+	}
+	return true
+}
+
+type ServiceAdvertisedIPs struct {
+	service       *v1core.Service
+	advertisedIPs []string
+	annotations   map[string]string
+}
+
+func Test_getVIPsForService(t *testing.T) {
+	services := map[string]*v1core.Service{
+		"cluster": {
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "svc-cluster",
+			},
+			Spec: v1core.ServiceSpec{
+				Type:      "ClusterIP",
+				ClusterIP: "10.0.0.1",
+			},
+		},
+		"external": {
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "svc-external",
+			},
+			Spec: v1core.ServiceSpec{
+				Type:        "ClusterIP",
+				ClusterIP:   "10.0.0.1",
+				ExternalIPs: []string{"1.1.1.1"},
+			},
+		},
+		"nodeport": {
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "svc-nodeport",
+			},
+			Spec: v1core.ServiceSpec{
+				Type:        "NodePort",
+				ClusterIP:   "10.0.0.1",
+				ExternalIPs: []string{"1.1.1.1"},
+			},
+		},
+		"loadbalancer": {
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "svc-loadbalancer",
+			},
+			Spec: v1core.ServiceSpec{
+				Type:      "LoadBalancer",
+				ClusterIP: "10.0.0.1",
+				// External IPs are ignored since LoadBalancer services don't
+				// advertise external IPs.
+				ExternalIPs: []string{"1.1.1.1"},
+			},
+			Status: v1core.ServiceStatus{
+				LoadBalancer: v1core.LoadBalancerStatus{
+					Ingress: []v1core.LoadBalancerIngress{
+						{
+							IP: "10.0.255.1",
+						},
+						{
+							IP: "10.0.255.2",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	tests := []struct {
+		name string
+		// cluster, external, loadbalancer
+		advertiseSettings    [3]bool
+		serviceAdvertisedIPs []*ServiceAdvertisedIPs
+	}{
+		{
+			"advertise all IPs",
+			[3]bool{true, true, true},
+			[]*ServiceAdvertisedIPs{
+				{
+					services["cluster"],
+					[]string{"10.0.0.1"},
+					nil,
+				},
+				{
+					services["external"],
+					[]string{"10.0.0.1", "1.1.1.1"},
+					nil,
+				},
+				{
+					services["nodeport"],
+					[]string{"10.0.0.1", "1.1.1.1"},
+					nil,
+				},
+				{
+					services["loadbalancer"],
+					[]string{"10.0.0.1", "10.0.255.1", "10.0.255.2"},
+					nil,
+				},
+			},
+		},
+		{
+			"do not advertise any IPs",
+			[3]bool{false, false, false},
+			[]*ServiceAdvertisedIPs{
+				{
+					services["cluster"],
+					[]string{},
+					nil,
+				},
+				{
+					services["external"],
+					[]string{},
+					nil,
+				},
+				{
+					services["nodeport"],
+					[]string{},
+					nil,
+				},
+				{
+					services["loadbalancer"],
+					[]string{},
+					nil,
+				},
+			},
+		},
+		{
+			"advertise cluster IPs",
+			[3]bool{true, false, false},
+			[]*ServiceAdvertisedIPs{
+				{
+					services["cluster"],
+					[]string{"10.0.0.1"},
+					nil,
+				},
+				{
+					services["external"],
+					[]string{"10.0.0.1"},
+					nil,
+				},
+				{
+					services["nodeport"],
+					[]string{"10.0.0.1"},
+					nil,
+				},
+				{
+					services["loadbalancer"],
+					[]string{"10.0.0.1"},
+					nil,
+				},
+			},
+		},
+		{
+			"advertise external IPs",
+			[3]bool{false, true, false},
+			[]*ServiceAdvertisedIPs{
+				{
+					services["cluster"],
+					[]string{},
+					nil,
+				},
+				{
+					services["external"],
+					[]string{"1.1.1.1"},
+					nil,
+				},
+				{
+					services["nodeport"],
+					[]string{"1.1.1.1"},
+					nil,
+				},
+				{
+					services["loadbalancer"],
+					[]string{},
+					nil,
+				},
+			},
+		},
+		{
+			"advertise loadbalancer IPs",
+			[3]bool{false, false, true},
+			[]*ServiceAdvertisedIPs{
+				{
+					services["cluster"],
+					[]string{},
+					nil,
+				},
+				{
+					services["external"],
+					[]string{},
+					nil,
+				},
+				{
+					services["nodeport"],
+					[]string{},
+					nil,
+				},
+				{
+					services["loadbalancer"],
+					[]string{"10.0.255.1", "10.0.255.2"},
+					nil,
+				},
+			},
+		},
+		{
+			"opt in to advertise all IPs via annotations",
+			[3]bool{false, false, false},
+			[]*ServiceAdvertisedIPs{
+				{
+					services["cluster"],
+					[]string{"10.0.0.1"},
+					map[string]string{
+						svcAdvertiseClusterAnnotation:      "true",
+						svcAdvertiseExternalAnnotation:     "true",
+						svcAdvertiseLoadBalancerAnnotation: "true",
+					},
+				},
+				{
+					services["external"],
+					[]string{"10.0.0.1", "1.1.1.1"},
+					map[string]string{
+						svcAdvertiseClusterAnnotation:      "true",
+						svcAdvertiseExternalAnnotation:     "true",
+						svcAdvertiseLoadBalancerAnnotation: "true",
+					},
+				},
+				{
+					services["nodeport"],
+					[]string{"10.0.0.1", "1.1.1.1"},
+					map[string]string{
+						svcAdvertiseClusterAnnotation:      "true",
+						svcAdvertiseExternalAnnotation:     "true",
+						svcAdvertiseLoadBalancerAnnotation: "true",
+					},
+				},
+				{
+					services["loadbalancer"],
+					[]string{"10.0.0.1", "10.0.255.1", "10.0.255.2"},
+					map[string]string{
+						svcAdvertiseClusterAnnotation:      "true",
+						svcAdvertiseExternalAnnotation:     "true",
+						svcAdvertiseLoadBalancerAnnotation: "true",
+					},
+				},
+				{
+					// Special case to test svcAdvertiseLoadBalancerAnnotation vs legacy svcSkipLbIpsAnnotation
+					services["loadbalancer"],
+					[]string{"10.0.0.1"},
+					map[string]string{
+						svcAdvertiseClusterAnnotation:      "true",
+						svcAdvertiseExternalAnnotation:     "true",
+						svcAdvertiseLoadBalancerAnnotation: "true",
+						svcSkipLbIpsAnnotation:             "true",
+					},
+				},
+			},
+		},
+		{
+			"opt out to advertise any IPs via annotations",
+			[3]bool{true, true, true},
+			[]*ServiceAdvertisedIPs{
+				{
+					services["cluster"],
+					[]string{},
+					map[string]string{
+						svcAdvertiseClusterAnnotation:      "false",
+						svcAdvertiseExternalAnnotation:     "false",
+						svcAdvertiseLoadBalancerAnnotation: "false",
+					},
+				},
+				{
+					services["external"],
+					[]string{},
+					map[string]string{
+						svcAdvertiseClusterAnnotation:      "false",
+						svcAdvertiseExternalAnnotation:     "false",
+						svcAdvertiseLoadBalancerAnnotation: "false",
+					},
+				},
+				{
+					services["nodeport"],
+					[]string{},
+					map[string]string{
+						svcAdvertiseClusterAnnotation:      "false",
+						svcAdvertiseExternalAnnotation:     "false",
+						svcAdvertiseLoadBalancerAnnotation: "false",
+					},
+				},
+				{
+					services["loadbalancer"],
+					[]string{},
+					map[string]string{
+						svcAdvertiseClusterAnnotation:      "false",
+						svcAdvertiseExternalAnnotation:     "false",
+						svcAdvertiseLoadBalancerAnnotation: "false",
+					},
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		nrc := NetworkRoutingController{}
+		t.Run(test.name, func(t *testing.T) {
+			nrc.advertiseClusterIP = test.advertiseSettings[0]
+			nrc.advertiseExternalIP = test.advertiseSettings[1]
+			nrc.advertiseLoadBalancerIP = test.advertiseSettings[2]
+
+			for _, serviceAdvertisedIP := range test.serviceAdvertisedIPs {
+				clientset := fake.NewSimpleClientset()
+
+				if serviceAdvertisedIP.annotations != nil {
+					serviceAdvertisedIP.service.ObjectMeta.Annotations = serviceAdvertisedIP.annotations
+				}
+				svc, _ := clientset.CoreV1().Services("default").Create(serviceAdvertisedIP.service)
+				advertisedIPs, _, _ := nrc.getVIPsForService(svc, false)
+				t.Logf("AdvertisedIPs: %v\n", advertisedIPs)
+				if !Equal(serviceAdvertisedIP.advertisedIPs, advertisedIPs) {
+					t.Errorf("Advertised IPs are incorrect, got: %v, want: %v.", serviceAdvertisedIP.advertisedIPs, advertisedIPs)
+				}
+			}
+		})
+	}
+}

pkg/controllers/routing/network_routes_controller.go

@@ -40,18 +40,24 @@ const (
 	podSubnetsIPSetName  = "kube-router-pod-subnets"
 	nodeAddrsIPSetName   = "kube-router-node-ips"
 
-	nodeASNAnnotation                 = "kube-router.io/node.asn"
+	nodeASNAnnotation                  = "kube-router.io/node.asn"
-	pathPrependASNAnnotation          = "kube-router.io/path-prepend.as"
+	pathPrependASNAnnotation           = "kube-router.io/path-prepend.as"
-	pathPrependRepeatNAnnotation      = "kube-router.io/path-prepend.repeat-n"
+	pathPrependRepeatNAnnotation       = "kube-router.io/path-prepend.repeat-n"
-	peerASNAnnotation                 = "kube-router.io/peer.asns"
+	peerASNAnnotation                  = "kube-router.io/peer.asns"
-	peerIPAnnotation                  = "kube-router.io/peer.ips"
+	peerIPAnnotation                   = "kube-router.io/peer.ips"
-	peerPasswordAnnotation            = "kube-router.io/peer.passwords"
+	peerPasswordAnnotation             = "kube-router.io/peer.passwords"
-	peerPortAnnotation                = "kube-router.io/peer.ports"
+	peerPortAnnotation                 = "kube-router.io/peer.ports"
-	rrClientAnnotation                = "kube-router.io/rr.client"
+	rrClientAnnotation                 = "kube-router.io/rr.client"
-	rrServerAnnotation                = "kube-router.io/rr.server"
+	rrServerAnnotation                 = "kube-router.io/rr.server"
-	svcLocalAnnotation                = "kube-router.io/service.local"
+	svcLocalAnnotation                 = "kube-router.io/service.local"
-	bgpLocalAddressAnnotation         = "kube-router.io/bgp-local-addresses"
+	bgpLocalAddressAnnotation          = "kube-router.io/bgp-local-addresses"
-	LeaderElectionRecordAnnotationKey = "control-plane.alpha.kubernetes.io/leader"
+	svcAdvertiseClusterAnnotation      = "kube-router.io/service.advertise.clusterip"
+	svcAdvertiseExternalAnnotation     = "kube-router.io/service.advertise.externalip"
+	svcAdvertiseLoadBalancerAnnotation = "kube-router.io/service.advertise.loadbalancerip"
+	LeaderElectionRecordAnnotationKey  = "control-plane.alpha.kubernetes.io/leader"
+
+	// Deprecated: use kube-router.io/service.advertise.loadbalancer instead
+	svcSkipLbIpsAnnotation = "kube-router.io/service.skiplbips"
 )
 
 // NetworkRoutingController is struct to hold necessary information required by controller

pkg/controllers/routing/network_routes_controller_test.go

@@ -446,7 +446,7 @@ func Test_advertiseExternalIPs(t *testing.T) {
 					ObjectMeta: metav1.ObjectMeta{
 						Name: "svc-1",
 						Annotations: map[string]string{
-							"kube-router.io/service.skiplbips": "true",
+							svcSkipLbIpsAnnotation: "true",
 						},
 					},
 					Spec: v1core.ServiceSpec{
@@ -520,6 +520,352 @@ func Test_advertiseExternalIPs(t *testing.T) {
 	}
 }
 
+func Test_advertiseAnnotationOptOut(t *testing.T) {
+	testcases := []struct {
+		name             string
+		nrc              *NetworkRoutingController
+		existingServices []*v1core.Service
+		// the key is the subnet from the watch event
+		watchEvents map[string]bool
+	}{
+		{
+			"add bgp paths for all service IPs",
+			&NetworkRoutingController{
+				bgpServer: gobgp.NewBgpServer(),
+			},
+			[]*v1core.Service{
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "svc-1",
+					},
+					Spec: v1core.ServiceSpec{
+						Type:        "ClusterIP",
+						ClusterIP:   "10.0.0.1",
+						ExternalIPs: []string{"1.1.1.1"},
+					},
+				},
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "svc-2",
+					},
+					Spec: v1core.ServiceSpec{
+						Type:        "NodePort",
+						ClusterIP:   "10.0.0.2",
+						ExternalIPs: []string{"2.2.2.2", "3.3.3.3"},
+					},
+				},
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "svc-3",
+					},
+					Spec: v1core.ServiceSpec{
+						Type:      "LoadBalancer",
+						ClusterIP: "10.0.0.3",
+						// ignored since LoadBalancer services don't
+						// advertise external IPs.
+						ExternalIPs: []string{"4.4.4.4"},
+					},
+					Status: v1core.ServiceStatus{
+						LoadBalancer: v1core.LoadBalancerStatus{
+							Ingress: []v1core.LoadBalancerIngress{
+								{
+									IP: "10.0.255.1",
+								},
+								{
+									IP: "10.0.255.2",
+								},
+							},
+						},
+					},
+				},
+			},
+			map[string]bool{
+				"10.0.0.1/32":   true,
+				"10.0.0.2/32":   true,
+				"10.0.0.3/32":   true,
+				"1.1.1.1/32":    true,
+				"2.2.2.2/32":    true,
+				"3.3.3.3/32":    true,
+				"10.0.255.1/32": true,
+				"10.0.255.2/32": true,
+			},
+		},
+		{
+			"opt out to advertise any IPs via annotations",
+			&NetworkRoutingController{
+				bgpServer: gobgp.NewBgpServer(),
+			},
+			[]*v1core.Service{
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "svc-1",
+						Annotations: map[string]string{
+							svcAdvertiseClusterAnnotation:      "false",
+							svcAdvertiseExternalAnnotation:     "false",
+							svcAdvertiseLoadBalancerAnnotation: "false",
+						},
+					},
+					Spec: v1core.ServiceSpec{
+						Type:        "LoadBalancer",
+						ClusterIP:   "10.0.0.1",
+						ExternalIPs: []string{"1.1.1.1", "2.2.2.2"},
+					},
+					Status: v1core.ServiceStatus{
+						LoadBalancer: v1core.LoadBalancerStatus{
+							Ingress: []v1core.LoadBalancerIngress{
+								{
+									IP: "10.0.255.1",
+								},
+								{
+									IP: "10.0.255.2",
+								},
+							},
+						},
+					},
+				},
+			},
+			map[string]bool{},
+		},
+	}
+
+	for _, testcase := range testcases {
+		t.Run(testcase.name, func(t *testing.T) {
+			go testcase.nrc.bgpServer.Serve()
+			err := testcase.nrc.bgpServer.Start(&config.Global{
+				Config: config.GlobalConfig{
+					As:       1,
+					RouterId: "10.0.0.0",
+					Port:     10000,
+				},
+			})
+			if err != nil {
+				t.Fatalf("failed to start BGP server: %v", err)
+			}
+			defer testcase.nrc.bgpServer.Stop()
+			w := testcase.nrc.bgpServer.Watch(gobgp.WatchBestPath(false))
+
+			clientset := fake.NewSimpleClientset()
+			startInformersForRoutes(testcase.nrc, clientset)
+
+			err = createServices(clientset, testcase.existingServices)
+			if err != nil {
+				t.Fatalf("failed to create existing services: %v", err)
+			}
+
+			waitForListerWithTimeout(testcase.nrc.svcLister, time.Second*10, t)
+
+			// By default advertise all IPs
+			testcase.nrc.advertiseClusterIP = true
+			testcase.nrc.advertiseExternalIP = true
+			testcase.nrc.advertiseLoadBalancerIP = true
+
+			toAdvertise, toWithdraw, _ := testcase.nrc.getActiveVIPs()
+			testcase.nrc.advertiseVIPs(toAdvertise)
+			testcase.nrc.withdrawVIPs(toWithdraw)
+
+			watchEvents := waitForBGPWatchEventWithTimeout(time.Second*10, len(testcase.watchEvents), w, t)
+			for _, watchEvent := range watchEvents {
+				for _, path := range watchEvent.PathList {
+					if _, ok := testcase.watchEvents[path.GetNlri().String()]; ok {
+						continue
+					} else {
+						t.Errorf("got unexpected path: %v", path.GetNlri().String())
+					}
+				}
+			}
+		})
+	}
+}
+
+func Test_advertiseAnnotationOptIn(t *testing.T) {
+	testcases := []struct {
+		name             string
+		nrc              *NetworkRoutingController
+		existingServices []*v1core.Service
+		// the key is the subnet from the watch event
+		watchEvents map[string]bool
+	}{
+		{
+			"no bgp paths for any service IPs",
+			&NetworkRoutingController{
+				bgpServer: gobgp.NewBgpServer(),
+			},
+			[]*v1core.Service{
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "svc-1",
+					},
+					Spec: v1core.ServiceSpec{
+						Type:        "ClusterIP",
+						ClusterIP:   "10.0.0.1",
+						ExternalIPs: []string{"1.1.1.1"},
+					},
+				},
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "svc-2",
+					},
+					Spec: v1core.ServiceSpec{
+						Type:        "NodePort",
+						ClusterIP:   "10.0.0.2",
+						ExternalIPs: []string{"2.2.2.2", "3.3.3.3"},
+					},
+				},
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "svc-3",
+					},
+					Spec: v1core.ServiceSpec{
+						Type:      "LoadBalancer",
+						ClusterIP: "10.0.0.3",
+						// ignored since LoadBalancer services don't
+						// advertise external IPs.
+						ExternalIPs: []string{"4.4.4.4"},
+					},
+					Status: v1core.ServiceStatus{
+						LoadBalancer: v1core.LoadBalancerStatus{
+							Ingress: []v1core.LoadBalancerIngress{
+								{
+									IP: "10.0.255.1",
+								},
+								{
+									IP: "10.0.255.2",
+								},
+							},
+						},
+					},
+				},
+			},
+			map[string]bool{},
+		},
+		{
+			"opt in to advertise all IPs via annotations",
+			&NetworkRoutingController{
+				bgpServer: gobgp.NewBgpServer(),
+			},
+			[]*v1core.Service{
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "svc-1",
+						Annotations: map[string]string{
+							svcAdvertiseClusterAnnotation:      "true",
+							svcAdvertiseExternalAnnotation:     "true",
+							svcAdvertiseLoadBalancerAnnotation: "true",
+						},
+					},
+					Spec: v1core.ServiceSpec{
+						Type:        "ClusterIP",
+						ClusterIP:   "10.0.0.1",
+						ExternalIPs: []string{"1.1.1.1"},
+					},
+				},
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "svc-2",
+						Annotations: map[string]string{
+							svcAdvertiseClusterAnnotation:      "true",
+							svcAdvertiseExternalAnnotation:     "true",
+							svcAdvertiseLoadBalancerAnnotation: "true",
+						},
+					},
+					Spec: v1core.ServiceSpec{
+						Type:        "NodePort",
+						ClusterIP:   "10.0.0.2",
+						ExternalIPs: []string{"2.2.2.2", "3.3.3.3"},
+					},
+				},
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "svc-3",
+						Annotations: map[string]string{
+							svcAdvertiseClusterAnnotation:      "true",
+							svcAdvertiseExternalAnnotation:     "true",
+							svcAdvertiseLoadBalancerAnnotation: "true",
+						},
+					},
+					Spec: v1core.ServiceSpec{
+						Type:      "LoadBalancer",
+						ClusterIP: "10.0.0.3",
+						// ignored since LoadBalancer services don't
+						// advertise external IPs.
+						ExternalIPs: []string{"4.4.4.4"},
+					},
+					Status: v1core.ServiceStatus{
+						LoadBalancer: v1core.LoadBalancerStatus{
+							Ingress: []v1core.LoadBalancerIngress{
+								{
+									IP: "10.0.255.1",
+								},
+								{
+									IP: "10.0.255.2",
+								},
+							},
+						},
+					},
+				},
+			},
+			map[string]bool{
+				"10.0.0.1/32":   true,
+				"10.0.0.2/32":   true,
+				"10.0.0.3/32":   true,
+				"1.1.1.1/32":    true,
+				"2.2.2.2/32":    true,
+				"3.3.3.3/32":    true,
+				"10.0.255.1/32": true,
+				"10.0.255.2/32": true,
+			},
+		},
+	}
+
+	for _, testcase := range testcases {
+		t.Run(testcase.name, func(t *testing.T) {
+			go testcase.nrc.bgpServer.Serve()
+			err := testcase.nrc.bgpServer.Start(&config.Global{
+				Config: config.GlobalConfig{
+					As:       1,
+					RouterId: "10.0.0.0",
+					Port:     10000,
+				},
+			})
+			if err != nil {
+				t.Fatalf("failed to start BGP server: %v", err)
+			}
+			defer testcase.nrc.bgpServer.Stop()
+			w := testcase.nrc.bgpServer.Watch(gobgp.WatchBestPath(false))
+
+			clientset := fake.NewSimpleClientset()
+			startInformersForRoutes(testcase.nrc, clientset)
+
+			err = createServices(clientset, testcase.existingServices)
+			if err != nil {
+				t.Fatalf("failed to create existing services: %v", err)
+			}
+
+			waitForListerWithTimeout(testcase.nrc.svcLister, time.Second*10, t)
+
+			// By default do not advertise any IPs
+			testcase.nrc.advertiseClusterIP = false
+			testcase.nrc.advertiseExternalIP = false
+			testcase.nrc.advertiseLoadBalancerIP = false
+
+			toAdvertise, toWithdraw, _ := testcase.nrc.getActiveVIPs()
+			testcase.nrc.advertiseVIPs(toAdvertise)
+			testcase.nrc.withdrawVIPs(toWithdraw)
+
+			watchEvents := waitForBGPWatchEventWithTimeout(time.Second*10, len(testcase.watchEvents), w, t)
+			for _, watchEvent := range watchEvents {
+				for _, path := range watchEvent.PathList {
+					if _, ok := testcase.watchEvents[path.GetNlri().String()]; ok {
+						continue
+					} else {
+						t.Errorf("got unexpected path: %v", path.GetNlri().String())
+					}
+				}
+			}
+		})
+	}
+}
+
 func Test_nodeHasEndpointsForService(t *testing.T) {
 	testcases := []struct {
 		name             string