mirrors/kube-router
1
0
Fork 0
mirror of https://github.com/cloudnativelabs/kube-router.git synced 2025-10-05 06:51:05 +02:00
Code Issues Packages Projects Releases Wiki Activity

implement per-service annotations to control IP advertisment (#575)

Browse Source 
* add unit tests for implementing #75

Signed-off-by: Steven Armstrong <steven.armstrong@id.ethz.ch>

* integration tests for #75

Signed-off-by: Steven Armstrong <steven.armstrong@id.ethz.ch>

* update docs for #75

Signed-off-by: Steven Armstrong <steven.armstrong@id.ethz.ch>

* define new kube-router.io/service.advertise.* annotations

Signed-off-by: Steven Armstrong <steven.armstrong@id.ethz.ch>

* Implement per service annotations for advertising IPs.

Signed-off-by: Steven Armstrong <steven.armstrong@id.ethz.ch>

* more consistent annotation names

Signed-off-by: Steven Armstrong <steven.armstrong@id.ethz.ch>

* remove redundant tests

Signed-off-by: Steven Armstrong <steven.armstrong@id.ethz.ch>
This commit is contained in:
Steven Armstrong 2018-12-09 06:08:12 +01:00 committed by Murali Reddy
parent e5d599b14c
commit 1a30f9e2e1
5 changed files with 761 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
docs/user-guide.md
View File

@ -134,6 +134,39 @@ and if you want to move back to kube-proxy then clean up config done by kube-rou
and run kube-proxy with the configuration you have.
- [General Setup](/README.md#getting-started)
## Advertising IPs
kube-router can advertise Cluster, External and LoadBalancer IPs to BGP peers.
It does this by:
* locally adding the advertised IPs to the nodes' `kube-dummy-if` network interface
* advertising the IPs to its BGP peers
To set the default for all services use the `--advertise-cluster-ip`,
`--advertise-external-ip` and `--advertise-loadbalancer-ip` flags.
To selectively enable or disable this feature per-service use the
`kube-router.io/service.advertise.cluster`, `kube-router.io/service.advertise.external`
and `kube-router.io/service.advertise.loadbalancer` annotations.
e.g.:
`$ kubectl annotate service my-advertised-service "kube-router.io/service.advertise.cluster=true"`
`$ kubectl annotate service my-advertised-service "kube-router.io/service.advertise.external=true"`
`$ kubectl annotate service my-advertised-service "kube-router.io/service.advertise.loadbalancer=true"`
`$ kubectl annotate service my-non-advertised-service "kube-router.io/service.advertise.cluster=false"`
`$ kubectl annotate service my-non-advertised-service "kube-router.io/service.advertise.external=false"`
`$ kubectl annotate service my-non-advertised-service "kube-router.io/service.advertise.loadbalancer=false"`
By combining the flags with the per-service annotations you can choose either
a opt-in or opt-out strategy for advertising IPs.
Advertising LoadBalancer IPs works by inspecting the services
`status.loadBalancer.ingress` IPs that are set by external LoadBalancers like
for example MetalLb. This has been successfully tested together with
[MetalLB](https://github.com/google/metallb) in ARP mode.
## Hairpin Mode
Communication from a Pod that is behind a Service to its own ClusterIP:Port is
@ -216,25 +249,6 @@ For destination hashing scheduling use:
kubectl annotate service my-service "kube-router.io/service.scheduler=dh"
```
## LoadBalancer IPs
If you want to also advertise loadbalancer set IPs
(`status.loadBalancer.ingress` IPs), e.g. when using it with MetalLb,
add the `--advertise-loadbalancer-ip` flag (`false` by default).
To selectively disable this behaviour per-service, you can use
the `kube-router.io/service.skiplbips` annotation as e.g.:
`$ kubectl annotate service my-external-service "kube-router.io/service.skiplbips=true"`
In concrete, unless the Service is annotated as per above, the
`--advertise-loadbalancer-ip` flag will make Service's Ingress IP(s)
set by the LoadBalancer to:
* be locally added to nodes' `kube-dummy-if` network interface
* be advertised to BGP peers
FYI Above has been successfully tested together with
[MetalLB](https://github.com/google/metallb) in ARP mode.
## HostPort support
If you would like to use `HostPort` functionality below changes are required in the manifest.

30
pkg/controllers/routing/ecmp_vip.go
View File

@ -268,12 +268,9 @@ func (nrc *NetworkRoutingController) getLoadBalancerIps(svc *v1core.Service) []s
if svc.Spec.Type == "LoadBalancer" {
// skip headless services
if svc.Spec.ClusterIP != "None" && svc.Spec.ClusterIP != "" {
_, skiplbips := svc.ObjectMeta.Annotations["kube-router.io/service.skiplbips"]
if !skiplbips {
for _, lbIngress := range svc.Status.LoadBalancer.Ingress {
if len(lbIngress.IP) > 0 {
loadBalancerIpList = append(loadBalancerIpList, lbIngress.IP)
}
for _, lbIngress := range svc.Status.LoadBalancer.Ingress {
if len(lbIngress.IP) > 0 {
loadBalancerIpList = append(loadBalancerIpList, lbIngress.IP)
}
}
}
@ -313,6 +310,16 @@ func (nrc *NetworkRoutingController) getVIPs(onlyActiveEndpoints bool) ([]string
return toAdvertiseList, toWithdrawList, nil
}
func (nrc *NetworkRoutingController) shouldAdvertiseService(svc *v1core.Service, annotation string, defaultValue bool) bool {
returnValue := defaultValue
stringValue, exists := svc.Annotations[annotation]
if exists {
// Service annotations overrides defaults.
returnValue, _ = strconv.ParseBool(stringValue)
}
return returnValue
}
func (nrc *NetworkRoutingController) getVIPsForService(svc *v1core.Service, onlyActiveEndpoints bool) ([]string, []string, error) {
ipList := make([]string, 0)
var err error
@ -328,16 +335,21 @@ func (nrc *NetworkRoutingController) getVIPsForService(svc *v1core.Service, only
}
}
if nrc.advertiseClusterIP {
if nrc.shouldAdvertiseService(svc, svcAdvertiseClusterAnnotation, nrc.advertiseClusterIP) {
clusterIp := nrc.getClusterIp(svc)
if clusterIp != "" {
ipList = append(ipList, clusterIp)
}
}
if nrc.advertiseExternalIP {
if nrc.shouldAdvertiseService(svc, svcAdvertiseExternalAnnotation, nrc.advertiseExternalIP) {
ipList = append(ipList, nrc.getExternalIps(svc)...)
}
if nrc.advertiseLoadBalancerIP {
// Deprecated: Use service.advertise.loadbalancer=false instead of service.skiplbips.
_, skiplbips := svc.Annotations[svcSkipLbIpsAnnotation]
advertiseLoadBalancer := nrc.shouldAdvertiseService(svc, svcAdvertiseLoadBalancerAnnotation, nrc.advertiseLoadBalancerIP)
if advertiseLoadBalancer && !skiplbips {
ipList = append(ipList, nrc.getLoadBalancerIps(svc)...)
}

342
pkg/controllers/routing/ecmp_vip_test.go Normal file
View File

@ -0,0 +1,342 @@
package routing
import (
"testing"
v1core "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/fake"
)
// Compare 2 string slices by value.
func Equal(a, b []string) bool {
if len(a) != len(b) {
return false
}
for i, v := range a {
if v != b[i] {
return false
}
}
return true
}
type ServiceAdvertisedIPs struct {
service *v1core.Service
advertisedIPs []string
annotations map[string]string
}
func Test_getVIPsForService(t *testing.T) {
services := map[string]*v1core.Service{
"cluster": {
ObjectMeta: metav1.ObjectMeta{
Name: "svc-cluster",
},
Spec: v1core.ServiceSpec{
Type: "ClusterIP",
ClusterIP: "10.0.0.1",
},
},
"external": {
ObjectMeta: metav1.ObjectMeta{
Name: "svc-external",
},
Spec: v1core.ServiceSpec{
Type: "ClusterIP",
ClusterIP: "10.0.0.1",
ExternalIPs: []string{"1.1.1.1"},
},
},
"nodeport": {
ObjectMeta: metav1.ObjectMeta{
Name: "svc-nodeport",
},
Spec: v1core.ServiceSpec{
Type: "NodePort",
ClusterIP: "10.0.0.1",
ExternalIPs: []string{"1.1.1.1"},
},
},
"loadbalancer": {
ObjectMeta: metav1.ObjectMeta{
Name: "svc-loadbalancer",
},
Spec: v1core.ServiceSpec{
Type: "LoadBalancer",
ClusterIP: "10.0.0.1",
// External IPs are ignored since LoadBalancer services don't
// advertise external IPs.
ExternalIPs: []string{"1.1.1.1"},
},
Status: v1core.ServiceStatus{
LoadBalancer: v1core.LoadBalancerStatus{
Ingress: []v1core.LoadBalancerIngress{
{
IP: "10.0.255.1",
},
{
IP: "10.0.255.2",
},
},
},
},
},
}
tests := []struct {
name string
// cluster, external, loadbalancer
advertiseSettings [3]bool
serviceAdvertisedIPs []*ServiceAdvertisedIPs
}{
{
"advertise all IPs",
[3]bool{true, true, true},
[]*ServiceAdvertisedIPs{
{
services["cluster"],
[]string{"10.0.0.1"},
nil,
},
{
services["external"],
[]string{"10.0.0.1", "1.1.1.1"},
nil,
},
{
services["nodeport"],
[]string{"10.0.0.1", "1.1.1.1"},
nil,
},
{
services["loadbalancer"],
[]string{"10.0.0.1", "10.0.255.1", "10.0.255.2"},
nil,
},
},
},
{
"do not advertise any IPs",
[3]bool{false, false, false},
[]*ServiceAdvertisedIPs{
{
services["cluster"],
[]string{},
nil,
},
{
services["external"],
[]string{},
nil,
},
{
services["nodeport"],
[]string{},
nil,
},
{
services["loadbalancer"],
[]string{},
nil,
},
},
},
{
"advertise cluster IPs",
[3]bool{true, false, false},
[]*ServiceAdvertisedIPs{
{
services["cluster"],
[]string{"10.0.0.1"},
nil,
},
{
services["external"],
[]string{"10.0.0.1"},
nil,
},
{
services["nodeport"],
[]string{"10.0.0.1"},
nil,
},
{
services["loadbalancer"],
[]string{"10.0.0.1"},
nil,
},
},
},
{
"advertise external IPs",
[3]bool{false, true, false},
[]*ServiceAdvertisedIPs{
{
services["cluster"],
[]string{},
nil,
},
{
services["external"],
[]string{"1.1.1.1"},
nil,
},
{
services["nodeport"],
[]string{"1.1.1.1"},
nil,
},
{
services["loadbalancer"],
[]string{},
nil,
},
},
},
{
"advertise loadbalancer IPs",
[3]bool{false, false, true},
[]*ServiceAdvertisedIPs{
{
services["cluster"],
[]string{},
nil,
},
{
services["external"],
[]string{},
nil,
},
{
services["nodeport"],
[]string{},
nil,
},
{
services["loadbalancer"],
[]string{"10.0.255.1", "10.0.255.2"},
nil,
},
},
},
{
"opt in to advertise all IPs via annotations",
[3]bool{false, false, false},
[]*ServiceAdvertisedIPs{
{
services["cluster"],
[]string{"10.0.0.1"},
map[string]string{
svcAdvertiseClusterAnnotation: "true",
svcAdvertiseExternalAnnotation: "true",
svcAdvertiseLoadBalancerAnnotation: "true",
},
},
{
services["external"],
[]string{"10.0.0.1", "1.1.1.1"},
map[string]string{
svcAdvertiseClusterAnnotation: "true",
svcAdvertiseExternalAnnotation: "true",
svcAdvertiseLoadBalancerAnnotation: "true",
},
},
{
services["nodeport"],
[]string{"10.0.0.1", "1.1.1.1"},
map[string]string{
svcAdvertiseClusterAnnotation: "true",
svcAdvertiseExternalAnnotation: "true",
svcAdvertiseLoadBalancerAnnotation: "true",
},
},
{
services["loadbalancer"],
[]string{"10.0.0.1", "10.0.255.1", "10.0.255.2"},
map[string]string{
svcAdvertiseClusterAnnotation: "true",
svcAdvertiseExternalAnnotation: "true",
svcAdvertiseLoadBalancerAnnotation: "true",
},
},
{
// Special case to test svcAdvertiseLoadBalancerAnnotation vs legacy svcSkipLbIpsAnnotation
services["loadbalancer"],
[]string{"10.0.0.1"},
map[string]string{
svcAdvertiseClusterAnnotation: "true",
svcAdvertiseExternalAnnotation: "true",
svcAdvertiseLoadBalancerAnnotation: "true",
svcSkipLbIpsAnnotation: "true",
},
},
},
},
{
"opt out to advertise any IPs via annotations",
[3]bool{true, true, true},
[]*ServiceAdvertisedIPs{
{
services["cluster"],
[]string{},
map[string]string{
svcAdvertiseClusterAnnotation: "false",
svcAdvertiseExternalAnnotation: "false",
svcAdvertiseLoadBalancerAnnotation: "false",
},
},
{
services["external"],
[]string{},
map[string]string{
svcAdvertiseClusterAnnotation: "false",
svcAdvertiseExternalAnnotation: "false",
svcAdvertiseLoadBalancerAnnotation: "false",
},
},
{
services["nodeport"],
[]string{},
map[string]string{
svcAdvertiseClusterAnnotation: "false",
svcAdvertiseExternalAnnotation: "false",
svcAdvertiseLoadBalancerAnnotation: "false",
},
},
{
services["loadbalancer"],
[]string{},
map[string]string{
svcAdvertiseClusterAnnotation: "false",
svcAdvertiseExternalAnnotation: "false",
svcAdvertiseLoadBalancerAnnotation: "false",
},
},
},
},
}
for _, test := range tests {
nrc := NetworkRoutingController{}
t.Run(test.name, func(t *testing.T) {
nrc.advertiseClusterIP = test.advertiseSettings[0]
nrc.advertiseExternalIP = test.advertiseSettings[1]
nrc.advertiseLoadBalancerIP = test.advertiseSettings[2]
for _, serviceAdvertisedIP := range test.serviceAdvertisedIPs {
clientset := fake.NewSimpleClientset()
if serviceAdvertisedIP.annotations != nil {
serviceAdvertisedIP.service.ObjectMeta.Annotations = serviceAdvertisedIP.annotations
}
svc, _ := clientset.CoreV1().Services("default").Create(serviceAdvertisedIP.service)
advertisedIPs, _, _ := nrc.getVIPsForService(svc, false)
t.Logf("AdvertisedIPs: %v\n", advertisedIPs)
if !Equal(serviceAdvertisedIP.advertisedIPs, advertisedIPs) {
t.Errorf("Advertised IPs are incorrect, got: %v, want: %v.", serviceAdvertisedIP.advertisedIPs, advertisedIPs)
}
}
})
}
}

30
pkg/controllers/routing/network_routes_controller.go
View File

@ -40,18 +40,24 @@ const (
podSubnetsIPSetName = "kube-router-pod-subnets"
nodeAddrsIPSetName = "kube-router-node-ips"
nodeASNAnnotation = "kube-router.io/node.asn"
pathPrependASNAnnotation = "kube-router.io/path-prepend.as"
pathPrependRepeatNAnnotation = "kube-router.io/path-prepend.repeat-n"
peerASNAnnotation = "kube-router.io/peer.asns"
peerIPAnnotation = "kube-router.io/peer.ips"
peerPasswordAnnotation = "kube-router.io/peer.passwords"
peerPortAnnotation = "kube-router.io/peer.ports"
rrClientAnnotation = "kube-router.io/rr.client"
rrServerAnnotation = "kube-router.io/rr.server"
svcLocalAnnotation = "kube-router.io/service.local"
bgpLocalAddressAnnotation = "kube-router.io/bgp-local-addresses"
LeaderElectionRecordAnnotationKey = "control-plane.alpha.kubernetes.io/leader"
nodeASNAnnotation = "kube-router.io/node.asn"
pathPrependASNAnnotation = "kube-router.io/path-prepend.as"
pathPrependRepeatNAnnotation = "kube-router.io/path-prepend.repeat-n"
peerASNAnnotation = "kube-router.io/peer.asns"
peerIPAnnotation = "kube-router.io/peer.ips"
peerPasswordAnnotation = "kube-router.io/peer.passwords"
peerPortAnnotation = "kube-router.io/peer.ports"
rrClientAnnotation = "kube-router.io/rr.client"
rrServerAnnotation = "kube-router.io/rr.server"
svcLocalAnnotation = "kube-router.io/service.local"
bgpLocalAddressAnnotation = "kube-router.io/bgp-local-addresses"
svcAdvertiseClusterAnnotation = "kube-router.io/service.advertise.clusterip"
svcAdvertiseExternalAnnotation = "kube-router.io/service.advertise.externalip"
svcAdvertiseLoadBalancerAnnotation = "kube-router.io/service.advertise.loadbalancerip"
LeaderElectionRecordAnnotationKey = "control-plane.alpha.kubernetes.io/leader"
// Deprecated: use kube-router.io/service.advertise.loadbalancer instead
svcSkipLbIpsAnnotation = "kube-router.io/service.skiplbips"
)
// NetworkRoutingController is struct to hold necessary information required by controller

348
pkg/controllers/routing/network_routes_controller_test.go
View File

@ -446,7 +446,7 @@ func Test_advertiseExternalIPs(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{
Name: "svc-1",
Annotations: map[string]string{
"kube-router.io/service.skiplbips": "true",
svcSkipLbIpsAnnotation: "true",
},
},
Spec: v1core.ServiceSpec{
@ -520,6 +520,352 @@ func Test_advertiseExternalIPs(t *testing.T) {
}
}
func Test_advertiseAnnotationOptOut(t *testing.T) {
testcases := []struct {
name string
nrc *NetworkRoutingController
existingServices []*v1core.Service
// the key is the subnet from the watch event
watchEvents map[string]bool
}{
{
"add bgp paths for all service IPs",
&NetworkRoutingController{
bgpServer: gobgp.NewBgpServer(),
},
[]*v1core.Service{
{
ObjectMeta: metav1.ObjectMeta{
Name: "svc-1",
},
Spec: v1core.ServiceSpec{
Type: "ClusterIP",
ClusterIP: "10.0.0.1",
ExternalIPs: []string{"1.1.1.1"},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "svc-2",
},
Spec: v1core.ServiceSpec{
Type: "NodePort",
ClusterIP: "10.0.0.2",
ExternalIPs: []string{"2.2.2.2", "3.3.3.3"},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "svc-3",
},
Spec: v1core.ServiceSpec{
Type: "LoadBalancer",
ClusterIP: "10.0.0.3",
// ignored since LoadBalancer services don't
// advertise external IPs.
ExternalIPs: []string{"4.4.4.4"},
},
Status: v1core.ServiceStatus{
LoadBalancer: v1core.LoadBalancerStatus{
Ingress: []v1core.LoadBalancerIngress{
{
IP: "10.0.255.1",
},
{
IP: "10.0.255.2",
},
},
},
},
},
},
map[string]bool{
"10.0.0.1/32": true,
"10.0.0.2/32": true,
"10.0.0.3/32": true,
"1.1.1.1/32": true,
"2.2.2.2/32": true,
"3.3.3.3/32": true,
"10.0.255.1/32": true,
"10.0.255.2/32": true,
},
},
{
"opt out to advertise any IPs via annotations",
&NetworkRoutingController{
bgpServer: gobgp.NewBgpServer(),
},
[]*v1core.Service{
{
ObjectMeta: metav1.ObjectMeta{
Name: "svc-1",
Annotations: map[string]string{
svcAdvertiseClusterAnnotation: "false",
svcAdvertiseExternalAnnotation: "false",
svcAdvertiseLoadBalancerAnnotation: "false",
},
},
Spec: v1core.ServiceSpec{
Type: "LoadBalancer",
ClusterIP: "10.0.0.1",
ExternalIPs: []string{"1.1.1.1", "2.2.2.2"},
},
Status: v1core.ServiceStatus{
LoadBalancer: v1core.LoadBalancerStatus{
Ingress: []v1core.LoadBalancerIngress{
{
IP: "10.0.255.1",
},
{
IP: "10.0.255.2",
},
},
},
},
},
},
map[string]bool{},
},
}
for _, testcase := range testcases {
t.Run(testcase.name, func(t *testing.T) {
go testcase.nrc.bgpServer.Serve()
err := testcase.nrc.bgpServer.Start(&config.Global{
Config: config.GlobalConfig{
As: 1,
RouterId: "10.0.0.0",
Port: 10000,
},
})
if err != nil {
t.Fatalf("failed to start BGP server: %v", err)
}
defer testcase.nrc.bgpServer.Stop()
w := testcase.nrc.bgpServer.Watch(gobgp.WatchBestPath(false))
clientset := fake.NewSimpleClientset()
startInformersForRoutes(testcase.nrc, clientset)
err = createServices(clientset, testcase.existingServices)
if err != nil {
t.Fatalf("failed to create existing services: %v", err)
}
waitForListerWithTimeout(testcase.nrc.svcLister, time.Second*10, t)
// By default advertise all IPs
testcase.nrc.advertiseClusterIP = true
testcase.nrc.advertiseExternalIP = true
testcase.nrc.advertiseLoadBalancerIP = true
toAdvertise, toWithdraw, _ := testcase.nrc.getActiveVIPs()
testcase.nrc.advertiseVIPs(toAdvertise)
testcase.nrc.withdrawVIPs(toWithdraw)
watchEvents := waitForBGPWatchEventWithTimeout(time.Second*10, len(testcase.watchEvents), w, t)
for _, watchEvent := range watchEvents {
for _, path := range watchEvent.PathList {
if _, ok := testcase.watchEvents[path.GetNlri().String()]; ok {
continue
} else {
t.Errorf("got unexpected path: %v", path.GetNlri().String())
}
}
}
})
}
}
func Test_advertiseAnnotationOptIn(t *testing.T) {
testcases := []struct {
name string
nrc *NetworkRoutingController
existingServices []*v1core.Service
// the key is the subnet from the watch event
watchEvents map[string]bool
}{
{
"no bgp paths for any service IPs",
&NetworkRoutingController{
bgpServer: gobgp.NewBgpServer(),
},
[]*v1core.Service{
{
ObjectMeta: metav1.ObjectMeta{
Name: "svc-1",
},
Spec: v1core.ServiceSpec{
Type: "ClusterIP",
ClusterIP: "10.0.0.1",
ExternalIPs: []string{"1.1.1.1"},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "svc-2",
},
Spec: v1core.ServiceSpec{
Type: "NodePort",
ClusterIP: "10.0.0.2",
ExternalIPs: []string{"2.2.2.2", "3.3.3.3"},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "svc-3",
},
Spec: v1core.ServiceSpec{
Type: "LoadBalancer",
ClusterIP: "10.0.0.3",
// ignored since LoadBalancer services don't
// advertise external IPs.
ExternalIPs: []string{"4.4.4.4"},
},
Status: v1core.ServiceStatus{
LoadBalancer: v1core.LoadBalancerStatus{
Ingress: []v1core.LoadBalancerIngress{
{
IP: "10.0.255.1",
},
{
IP: "10.0.255.2",
},
},
},
},
},
},
map[string]bool{},
},
{
"opt in to advertise all IPs via annotations",
&NetworkRoutingController{
bgpServer: gobgp.NewBgpServer(),
},
[]*v1core.Service{
{
ObjectMeta: metav1.ObjectMeta{
Name: "svc-1",
Annotations: map[string]string{
svcAdvertiseClusterAnnotation: "true",
svcAdvertiseExternalAnnotation: "true",
svcAdvertiseLoadBalancerAnnotation: "true",
},
},
Spec: v1core.ServiceSpec{
Type: "ClusterIP",
ClusterIP: "10.0.0.1",
ExternalIPs: []string{"1.1.1.1"},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "svc-2",
Annotations: map[string]string{
svcAdvertiseClusterAnnotation: "true",
svcAdvertiseExternalAnnotation: "true",
svcAdvertiseLoadBalancerAnnotation: "true",
},
},
Spec: v1core.ServiceSpec{
Type: "NodePort",
ClusterIP: "10.0.0.2",
ExternalIPs: []string{"2.2.2.2", "3.3.3.3"},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "svc-3",
Annotations: map[string]string{
svcAdvertiseClusterAnnotation: "true",
svcAdvertiseExternalAnnotation: "true",
svcAdvertiseLoadBalancerAnnotation: "true",
},
},
Spec: v1core.ServiceSpec{
Type: "LoadBalancer",
ClusterIP: "10.0.0.3",
// ignored since LoadBalancer services don't
// advertise external IPs.
ExternalIPs: []string{"4.4.4.4"},
},
Status: v1core.ServiceStatus{
LoadBalancer: v1core.LoadBalancerStatus{
Ingress: []v1core.LoadBalancerIngress{
{
IP: "10.0.255.1",
},
{
IP: "10.0.255.2",
},
},
},
},
},
},
map[string]bool{
"10.0.0.1/32": true,
"10.0.0.2/32": true,
"10.0.0.3/32": true,
"1.1.1.1/32": true,
"2.2.2.2/32": true,
"3.3.3.3/32": true,
"10.0.255.1/32": true,
"10.0.255.2/32": true,
},
},
}
for _, testcase := range testcases {
t.Run(testcase.name, func(t *testing.T) {
go testcase.nrc.bgpServer.Serve()
err := testcase.nrc.bgpServer.Start(&config.Global{
Config: config.GlobalConfig{
As: 1,
RouterId: "10.0.0.0",
Port: 10000,
},
})
if err != nil {
t.Fatalf("failed to start BGP server: %v", err)
}
defer testcase.nrc.bgpServer.Stop()
w := testcase.nrc.bgpServer.Watch(gobgp.WatchBestPath(false))
clientset := fake.NewSimpleClientset()
startInformersForRoutes(testcase.nrc, clientset)
err = createServices(clientset, testcase.existingServices)
if err != nil {
t.Fatalf("failed to create existing services: %v", err)
}
waitForListerWithTimeout(testcase.nrc.svcLister, time.Second*10, t)
// By default do not advertise any IPs
testcase.nrc.advertiseClusterIP = false
testcase.nrc.advertiseExternalIP = false
testcase.nrc.advertiseLoadBalancerIP = false
toAdvertise, toWithdraw, _ := testcase.nrc.getActiveVIPs()
testcase.nrc.advertiseVIPs(toAdvertise)
testcase.nrc.withdrawVIPs(toWithdraw)
watchEvents := waitForBGPWatchEventWithTimeout(time.Second*10, len(testcase.watchEvents), w, t)
for _, watchEvent := range watchEvents {
for _, path := range watchEvent.PathList {
if _, ok := testcase.watchEvents[path.GetNlri().String()]; ok {
continue
} else {
t.Errorf("got unexpected path: %v", path.GetNlri().String())
}
}
}
})
}
}
func Test_nodeHasEndpointsForService(t *testing.T) {
testcases := []struct {
name string