mirror of
https://github.com/cloudnativelabs/kube-router.git
synced 2025-09-28 11:31:05 +02:00
06f5f8babf
Do the necessary to update kube-router to a new major version following upstream documentation: https://go.dev/doc/modules/major-version
673 lines
25 KiB
Go
673 lines
25 KiB
Go
package proxy
|
|
|
|
import (
|
|
"bytes"
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
"strconv"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/cloudnativelabs/kube-router/v2/pkg/metrics"
|
|
"github.com/cloudnativelabs/kube-router/v2/pkg/utils"
|
|
"github.com/moby/ipvs"
|
|
"github.com/vishvananda/netlink"
|
|
"k8s.io/apimachinery/pkg/util/sets"
|
|
"k8s.io/klog/v2"
|
|
)
|
|
|
|
// sync the ipvs service and server details configured to reflect the desired state of Kubernetes services
|
|
// and endpoints as learned from services and endpoints information from the api server
|
|
func (nsc *NetworkServicesController) syncIpvsServices(serviceInfoMap serviceInfoMap,
|
|
endpointsInfoMap endpointsInfoMap) error {
|
|
start := time.Now()
|
|
defer func() {
|
|
endTime := time.Since(start)
|
|
if nsc.MetricsEnabled {
|
|
metrics.ControllerIpvsServicesSyncTime.Observe(endTime.Seconds())
|
|
}
|
|
klog.V(1).Infof("sync ipvs services took %v", endTime)
|
|
}()
|
|
|
|
var err error
|
|
var syncErrors bool
|
|
|
|
// map to track all active IPVS services and servers that are setup during sync of
|
|
// cluster IP, nodeport and external IP services
|
|
activeServiceEndpointMap := make(map[string][]string)
|
|
|
|
err = nsc.setupClusterIPServices(serviceInfoMap, endpointsInfoMap, activeServiceEndpointMap)
|
|
if err != nil {
|
|
syncErrors = true
|
|
klog.Errorf("Error setting up IPVS services for service cluster IP's: %s", err.Error())
|
|
}
|
|
err = nsc.setupNodePortServices(serviceInfoMap, endpointsInfoMap, activeServiceEndpointMap)
|
|
if err != nil {
|
|
syncErrors = true
|
|
klog.Errorf("Error setting up IPVS services for service nodeport's: %s", err.Error())
|
|
}
|
|
err = nsc.setupExternalIPServices(serviceInfoMap, endpointsInfoMap, activeServiceEndpointMap)
|
|
if err != nil {
|
|
syncErrors = true
|
|
klog.Errorf("Error setting up IPVS services for service external IP's and load balancer IP's: %s",
|
|
err.Error())
|
|
}
|
|
err = nsc.cleanupStaleVIPs(activeServiceEndpointMap)
|
|
if err != nil {
|
|
syncErrors = true
|
|
klog.Errorf("Error cleaning up stale VIP's configured on the dummy interface: %s", err.Error())
|
|
}
|
|
err = nsc.cleanupStaleIPVSConfig(activeServiceEndpointMap)
|
|
if err != nil {
|
|
syncErrors = true
|
|
klog.Errorf("Error cleaning up stale IPVS services and servers: %s", err.Error())
|
|
}
|
|
|
|
nsc.cleanupStaleMetrics(activeServiceEndpointMap)
|
|
|
|
err = nsc.syncIpvsFirewall()
|
|
if err != nil {
|
|
syncErrors = true
|
|
klog.Errorf("Error syncing ipvs svc iptables rules to permit traffic to service VIP's: %s", err.Error())
|
|
}
|
|
err = nsc.setupForDSR(serviceInfoMap)
|
|
if err != nil {
|
|
syncErrors = true
|
|
klog.Errorf("Error setting up necessary policy based routing configuration needed for "+
|
|
"direct server return: %s", err.Error())
|
|
}
|
|
|
|
if syncErrors {
|
|
klog.V(1).Info("One or more errors encountered during sync of IPVS services and servers " +
|
|
"to desired state")
|
|
} else {
|
|
klog.V(1).Info("IPVS servers and services are synced to desired state")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (nsc *NetworkServicesController) setupClusterIPServices(serviceInfoMap serviceInfoMap,
|
|
endpointsInfoMap endpointsInfoMap, activeServiceEndpointMap map[string][]string) error {
|
|
ipvsSvcs, err := nsc.ln.ipvsGetServices()
|
|
if err != nil {
|
|
return errors.New("Failed get list of IPVS services due to: " + err.Error())
|
|
}
|
|
for k, svc := range serviceInfoMap {
|
|
protocol := convertSvcProtoToSysCallProto(svc.protocol)
|
|
|
|
endpoints := endpointsInfoMap[k]
|
|
dummyVipInterface, err := nsc.ln.getKubeDummyInterface()
|
|
if err != nil {
|
|
return errors.New("Failed creating dummy interface: " + err.Error())
|
|
}
|
|
// assign cluster IP of the service to the dummy interface so that its routable from the pod's on the node
|
|
err = nsc.ln.ipAddrAdd(dummyVipInterface, svc.clusterIP.String(), true)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
// create IPVS service for the service to be exposed through the cluster ip
|
|
ipvsClusterVipSvc, err := nsc.ln.ipvsAddService(ipvsSvcs, svc.clusterIP, protocol, uint16(svc.port),
|
|
svc.sessionAffinity, svc.sessionAffinityTimeoutSeconds, svc.scheduler, svc.flags)
|
|
if err != nil {
|
|
klog.Errorf("Failed to create ipvs service for cluster ip: %s", err.Error())
|
|
continue
|
|
}
|
|
var clusterServiceID = generateIPPortID(svc.clusterIP.String(), svc.protocol, strconv.Itoa(svc.port))
|
|
activeServiceEndpointMap[clusterServiceID] = make([]string, 0)
|
|
|
|
// add IPVS remote server to the IPVS service
|
|
for _, endpoint := range endpoints {
|
|
dst := ipvs.Destination{
|
|
Address: net.ParseIP(endpoint.ip),
|
|
AddressFamily: syscall.AF_INET,
|
|
Port: uint16(endpoint.port),
|
|
Weight: 1,
|
|
}
|
|
// Conditions on which to add an endpoint on this node:
|
|
// 1) Service is not a local service
|
|
// 2) Service is a local service, but has no active endpoints on this node
|
|
// 3) Service is a local service, has active endpoints on this node, and this endpoint is one of them
|
|
if svc.local {
|
|
if hasActiveEndpoints(endpoints) && !endpoint.isLocal {
|
|
continue
|
|
}
|
|
}
|
|
|
|
err := nsc.ln.ipvsAddServer(ipvsClusterVipSvc, &dst)
|
|
if err != nil {
|
|
klog.Errorf(err.Error())
|
|
} else {
|
|
activeServiceEndpointMap[clusterServiceID] = append(activeServiceEndpointMap[clusterServiceID],
|
|
generateEndpointID(endpoint.ip, strconv.Itoa(endpoint.port)))
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (nsc *NetworkServicesController) setupNodePortServices(serviceInfoMap serviceInfoMap,
|
|
endpointsInfoMap endpointsInfoMap, activeServiceEndpointMap map[string][]string) error {
|
|
ipvsSvcs, err := nsc.ln.ipvsGetServices()
|
|
if err != nil {
|
|
return errors.New("Failed get list of IPVS services due to: " + err.Error())
|
|
}
|
|
for k, svc := range serviceInfoMap {
|
|
protocol := convertSvcProtoToSysCallProto(svc.protocol)
|
|
|
|
if svc.nodePort == 0 {
|
|
// service is not NodePort type
|
|
continue
|
|
}
|
|
endpoints := endpointsInfoMap[k]
|
|
if svc.local && !hasActiveEndpoints(endpoints) {
|
|
klog.V(1).Infof("Skipping setting up NodePort service %s/%s as it does not have active endpoints",
|
|
svc.namespace, svc.name)
|
|
continue
|
|
}
|
|
|
|
// create IPVS service for the service to be exposed through the nodeport
|
|
var ipvsNodeportSvcs []*ipvs.Service
|
|
|
|
var nodeServiceIds []string
|
|
|
|
if nsc.nodeportBindOnAllIP {
|
|
// bind on all interfaces instead
|
|
addrs, err := getAllLocalIPs()
|
|
|
|
if err != nil {
|
|
klog.Errorf("Could not get list of system addresses for ipvs services: %s", err.Error())
|
|
continue
|
|
}
|
|
|
|
if len(addrs) == 0 {
|
|
klog.Errorf("No IP addresses returned for nodeport service creation!")
|
|
continue
|
|
}
|
|
|
|
ipvsNodeportSvcs = make([]*ipvs.Service, len(addrs))
|
|
nodeServiceIds = make([]string, len(addrs))
|
|
|
|
for i, addr := range addrs {
|
|
ipvsNodeportSvcs[i], err = nsc.ln.ipvsAddService(ipvsSvcs, addr.IP, protocol, uint16(svc.nodePort),
|
|
svc.sessionAffinity, svc.sessionAffinityTimeoutSeconds, svc.scheduler, svc.flags)
|
|
if err != nil {
|
|
klog.Errorf("Failed to create ipvs service for node port due to: %s", err.Error())
|
|
continue
|
|
}
|
|
|
|
nodeServiceIds[i] = generateIPPortID(addr.IP.String(), svc.protocol, strconv.Itoa(svc.nodePort))
|
|
activeServiceEndpointMap[nodeServiceIds[i]] = make([]string, 0)
|
|
}
|
|
} else {
|
|
ipvsNodeportSvcs = make([]*ipvs.Service, 1)
|
|
ipvsNodeportSvcs[0], err = nsc.ln.ipvsAddService(ipvsSvcs, nsc.nodeIP, protocol, uint16(svc.nodePort),
|
|
svc.sessionAffinity, svc.sessionAffinityTimeoutSeconds, svc.scheduler, svc.flags)
|
|
if err != nil {
|
|
klog.Errorf("Failed to create ipvs service for node port due to: %s", err.Error())
|
|
continue
|
|
}
|
|
|
|
nodeServiceIds = make([]string, 1)
|
|
nodeServiceIds[0] = generateIPPortID(nsc.nodeIP.String(), svc.protocol, strconv.Itoa(svc.nodePort))
|
|
activeServiceEndpointMap[nodeServiceIds[0]] = make([]string, 0)
|
|
}
|
|
|
|
for _, endpoint := range endpoints {
|
|
dst := ipvs.Destination{
|
|
Address: net.ParseIP(endpoint.ip),
|
|
AddressFamily: syscall.AF_INET,
|
|
Port: uint16(endpoint.port),
|
|
Weight: 1,
|
|
}
|
|
for i := 0; i < len(ipvsNodeportSvcs); i++ {
|
|
if !svc.local || (svc.local && endpoint.isLocal) {
|
|
err := nsc.ln.ipvsAddServer(ipvsNodeportSvcs[i], &dst)
|
|
if err != nil {
|
|
klog.Errorf(err.Error())
|
|
} else {
|
|
activeServiceEndpointMap[nodeServiceIds[i]] =
|
|
append(activeServiceEndpointMap[nodeServiceIds[i]],
|
|
generateEndpointID(endpoint.ip, strconv.Itoa(endpoint.port)))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (nsc *NetworkServicesController) setupExternalIPServices(serviceInfoMap serviceInfoMap,
|
|
endpointsInfoMap endpointsInfoMap, activeServiceEndpointMap map[string][]string) error {
|
|
for k, svc := range serviceInfoMap {
|
|
endpoints := endpointsInfoMap[k]
|
|
|
|
extIPSet := sets.NewString(svc.externalIPs...)
|
|
if !svc.skipLbIps {
|
|
extIPSet = extIPSet.Union(sets.NewString(svc.loadBalancerIPs...))
|
|
}
|
|
|
|
if extIPSet.Len() == 0 {
|
|
// service is not LoadBalancer type and no external IP's are configured
|
|
continue
|
|
}
|
|
|
|
if svc.local && !hasActiveEndpoints(endpoints) {
|
|
klog.V(1).Infof("Skipping setting up IPVS service for external IP and LoadBalancer IP "+
|
|
"for the service %s/%s as it does not have active endpoints\n", svc.namespace, svc.name)
|
|
continue
|
|
}
|
|
for _, externalIP := range extIPSet.List() {
|
|
var externalIPServiceID string
|
|
if svc.directServerReturn && svc.directServerReturnMethod == tunnelInterfaceType {
|
|
// for a DSR service, do the work necessary to set up the IPVS service for DSR, then use the FW mark
|
|
// that was generated to add this external IP to the activeServiceEndpointMap
|
|
if err := nsc.setupExternalIPForDSRService(svc, externalIP, endpoints); err != nil {
|
|
return fmt.Errorf("failed to setup DSR endpoint %s: %v", externalIP, err)
|
|
}
|
|
fwMark := nsc.lookupFWMarkByService(externalIP, svc.protocol, fmt.Sprint(svc.port))
|
|
externalIPServiceID = fmt.Sprint(fwMark)
|
|
} else {
|
|
// for a non-DSR service, do the work necessary to setup the IPVS service, then use its IP, protocol,
|
|
// and port to add this external IP to the activeServiceEndpointMap
|
|
if err := nsc.setupExternalIPForService(svc, externalIP, endpoints); err != nil {
|
|
return fmt.Errorf("failed to setup service endpoint %s: %v", externalIP, err)
|
|
}
|
|
externalIPServiceID = generateIPPortID(externalIP, svc.protocol, strconv.Itoa(svc.port))
|
|
}
|
|
|
|
// add external service to the activeServiceEndpointMap by its externalIPServiceID. In this case,
|
|
// externalIPServiceID is a little confusing because in the case of DSR services it is the FW Mark that is
|
|
// generated for it, and for non-DSR services it is the combination of: ip + "-" + protocol + "-" + port
|
|
// TODO: remove the difference between DSR and non-DSR services and make a standard
|
|
activeServiceEndpointMap[externalIPServiceID] = make([]string, 0)
|
|
for _, endpoint := range endpoints {
|
|
if !svc.local || (svc.local && endpoint.isLocal) {
|
|
activeServiceEndpointMap[externalIPServiceID] =
|
|
append(activeServiceEndpointMap[externalIPServiceID],
|
|
generateEndpointID(endpoint.ip, strconv.Itoa(endpoint.port)))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// setupExternalIPForService does the basic work to setup a non-DSR based external IP for service. This includes adding
|
|
// the IPVS service to the host if it is missing, and setting up the dummy interface to be able to receive traffic on
|
|
// the node.
|
|
func (nsc *NetworkServicesController) setupExternalIPForService(svc *serviceInfo, externalIP string,
|
|
endpoints []endpointsInfo) error {
|
|
// Get everything we need to get setup to process the external IP
|
|
protocol := convertSvcProtoToSysCallProto(svc.protocol)
|
|
dummyVipInterface, err := nsc.ln.getKubeDummyInterface()
|
|
if err != nil {
|
|
return fmt.Errorf("failed creating dummy interface: %v", err)
|
|
}
|
|
ipvsSvcs, err := nsc.ln.ipvsGetServices()
|
|
if err != nil {
|
|
return fmt.Errorf("failed get list of IPVS services due to: %v", err)
|
|
}
|
|
|
|
// ensure director with vip assigned
|
|
err = nsc.ln.ipAddrAdd(dummyVipInterface, externalIP, true)
|
|
if err != nil && err.Error() != IfaceHasAddr {
|
|
return fmt.Errorf("failed to assign external ip %s to dummy interface %s due to %v",
|
|
externalIP, KubeDummyIf, err)
|
|
}
|
|
|
|
// create IPVS service for the service to be exposed through the external ip
|
|
ipvsExternalIPSvc, err := nsc.ln.ipvsAddService(ipvsSvcs, net.ParseIP(externalIP), protocol,
|
|
uint16(svc.port), svc.sessionAffinity, svc.sessionAffinityTimeoutSeconds, svc.scheduler, svc.flags)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create ipvs service for external ip: %s due to %v",
|
|
externalIP, err)
|
|
}
|
|
|
|
// ensure there is NO iptables mangle table rule to FW mark the packet
|
|
fwMark := nsc.lookupFWMarkByService(externalIP, svc.protocol, strconv.Itoa(svc.port))
|
|
switch {
|
|
case fwMark == 0:
|
|
klog.V(2).Infof("no FW mark found for service, nothing to cleanup")
|
|
case fwMark != 0:
|
|
klog.V(2).Infof("the following service '%s:%s:%d' had fwMark associated with it: %d doing "+
|
|
"additional cleanup", externalIP, svc.protocol, svc.port, fwMark)
|
|
if err = nsc.cleanupDSRService(fwMark); err != nil {
|
|
return fmt.Errorf("failed to cleanup DSR service: %v", err)
|
|
}
|
|
}
|
|
|
|
// add pod endpoints to the IPVS service
|
|
for _, endpoint := range endpoints {
|
|
// if this specific endpoint isn't local, there is nothing for us to do and we can go to the next record
|
|
if svc.local && !endpoint.isLocal {
|
|
continue
|
|
}
|
|
|
|
// create the basic IPVS destination record
|
|
dst := ipvs.Destination{
|
|
Address: net.ParseIP(endpoint.ip),
|
|
AddressFamily: syscall.AF_INET,
|
|
Port: uint16(endpoint.port),
|
|
Weight: 1,
|
|
}
|
|
|
|
if err = nsc.ln.ipvsAddServer(ipvsExternalIPSvc, &dst); err != nil {
|
|
return fmt.Errorf("unable to add destination %s to externalIP service %s: %v",
|
|
endpoint.ip, externalIP, err)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// setupExternalIPForDSRService does the basic setup necessary to set up an External IP service for DSR. This includes
|
|
// generating a unique FW mark for the service, setting up the mangle rules to apply the FW mark, setting up IPVS to
|
|
// work with the FW mark, and ensuring that the IP doesn't exist on the dummy interface so that the traffic doesn't
|
|
// accidentally ingress the packet and change it.
|
|
//
|
|
// For external IPs (which are meant for ingress traffic) configured for DSR, kube-router sets up IPVS services
|
|
// based on FWMARK to enable direct server return functionality. DSR requires a director without a VIP
|
|
// http://www.austintek.com/LVS/LVS-HOWTO/HOWTO/LVS-HOWTO.routing_to_VIP-less_director.html to avoid martian packets
|
|
func (nsc *NetworkServicesController) setupExternalIPForDSRService(svc *serviceInfo, externalIP string,
|
|
endpoints []endpointsInfo) error {
|
|
// Get everything we need to get setup to process the external IP
|
|
protocol := convertSvcProtoToSysCallProto(svc.protocol)
|
|
dummyVipInterface, err := nsc.ln.getKubeDummyInterface()
|
|
if err != nil {
|
|
return errors.New("Failed creating dummy interface: " + err.Error())
|
|
}
|
|
ipvsSvcs, err := nsc.ln.ipvsGetServices()
|
|
if err != nil {
|
|
return errors.New("Failed get list of IPVS services due to: " + err.Error())
|
|
}
|
|
|
|
fwMark, err := nsc.generateUniqueFWMark(externalIP, svc.protocol, strconv.Itoa(svc.port))
|
|
if err != nil {
|
|
return fmt.Errorf("failed to generate FW mark")
|
|
}
|
|
ipvsExternalIPSvc, err := nsc.ln.ipvsAddFWMarkService(ipvsSvcs, fwMark, protocol, uint16(svc.port),
|
|
svc.sessionAffinity, svc.sessionAffinityTimeoutSeconds, svc.scheduler, svc.flags)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create IPVS service for External IP: %s due to: %s",
|
|
externalIP, err.Error())
|
|
}
|
|
|
|
externalIPServiceID := fmt.Sprint(fwMark)
|
|
|
|
// ensure there is iptables mangle table rule to FWMARK the packet
|
|
err = setupMangleTableRule(externalIP, svc.protocol, strconv.Itoa(svc.port), externalIPServiceID,
|
|
nsc.dsrTCPMSS)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to setup mangle table rule to forward the traffic to external IP")
|
|
}
|
|
|
|
// ensure VIP less director. we dont assign VIP to any interface
|
|
err = nsc.ln.ipAddrDel(dummyVipInterface, externalIP)
|
|
if err != nil && err.Error() != IfaceHasNoAddr {
|
|
return fmt.Errorf("failed to delete external ip address from dummyVipInterface due to %v", err)
|
|
}
|
|
|
|
// do policy routing to deliver the packet locally so that IPVS can pick the packet
|
|
err = routeVIPTrafficToDirector("0x" + fmt.Sprintf("%x", fwMark))
|
|
if err != nil {
|
|
return fmt.Errorf("failed to setup ip rule to lookup traffic to external IP: %s through custom "+
|
|
"route table due to %v", externalIP, err)
|
|
}
|
|
|
|
// add pod endpoints to the IPVS service
|
|
for _, endpoint := range endpoints {
|
|
// if this specific endpoint isn't local, there is nothing for us to do and we can go to the next record
|
|
if svc.local && !endpoint.isLocal {
|
|
continue
|
|
}
|
|
|
|
// create the basic IPVS destination record
|
|
dst := ipvs.Destination{
|
|
Address: net.ParseIP(endpoint.ip),
|
|
AddressFamily: syscall.AF_INET,
|
|
ConnectionFlags: ipvs.ConnectionFlagTunnel,
|
|
Port: uint16(endpoint.port),
|
|
Weight: 1,
|
|
}
|
|
|
|
// add the destination for the IPVS service for this external IP
|
|
if err = nsc.ln.ipvsAddServer(ipvsExternalIPSvc, &dst); err != nil {
|
|
return fmt.Errorf("unable to add destination %s to externalIP service %s: %v",
|
|
endpoint.ip, externalIP, err)
|
|
}
|
|
|
|
// add the external IP to a virtual interface inside the pod so that the pod can receive it
|
|
if err = nsc.addDSRIPInsidePodNetNamespace(externalIP, endpoint.ip); err != nil {
|
|
return fmt.Errorf("unable to setup DSR receiver inside pod: %v", err)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (nsc *NetworkServicesController) setupForDSR(serviceInfoMap serviceInfoMap) error {
|
|
klog.V(1).Infof("Setting up policy routing required for Direct Server Return functionality.")
|
|
err := nsc.ln.setupPolicyRoutingForDSR()
|
|
if err != nil {
|
|
return errors.New("Failed setup PBR for DSR due to: " + err.Error())
|
|
}
|
|
klog.V(1).Infof("Custom routing table %s required for Direct Server Return is setup as expected.",
|
|
customDSRRouteTableName)
|
|
|
|
klog.V(1).Infof("Setting up custom route table required to add routes for external IP's.")
|
|
err = nsc.ln.setupRoutesForExternalIPForDSR(serviceInfoMap)
|
|
if err != nil {
|
|
klog.Errorf("failed setup custom routing table required to add routes for external IP's due to: %v",
|
|
err)
|
|
return fmt.Errorf("failed setup custom routing table required to add routes for external IP's due to: %v",
|
|
err)
|
|
}
|
|
klog.V(1).Infof("Custom routing table required for Direct Server Return (%s) is setup as expected.",
|
|
externalIPRouteTableName)
|
|
return nil
|
|
}
|
|
|
|
func (nsc *NetworkServicesController) cleanupStaleVIPs(activeServiceEndpointMap map[string][]string) error {
|
|
// cleanup stale IPs on dummy interface
|
|
klog.V(1).Info("Cleaning up if any, old service IPs on dummy interface")
|
|
// This represents "ip - protocol - port" that is created as the key to activeServiceEndpointMap in
|
|
// generateIPPortID()
|
|
const expectedServiceIDParts = 3
|
|
addrActive := make(map[string]bool)
|
|
for k := range activeServiceEndpointMap {
|
|
// verify active and its a generateIPPortID() type service
|
|
if strings.Contains(k, "-") {
|
|
parts := strings.SplitN(k, "-", expectedServiceIDParts)
|
|
addrActive[parts[0]] = true
|
|
}
|
|
}
|
|
|
|
dummyVipInterface, err := nsc.ln.getKubeDummyInterface()
|
|
if err != nil {
|
|
return errors.New("Failed creating dummy interface: " + err.Error())
|
|
}
|
|
var addrs []netlink.Addr
|
|
addrs, err = netlink.AddrList(dummyVipInterface, netlink.FAMILY_V4)
|
|
if err != nil {
|
|
return errors.New("Failed to list dummy interface IPs: " + err.Error())
|
|
}
|
|
for _, addr := range addrs {
|
|
isActive := addrActive[addr.IP.String()]
|
|
if !isActive {
|
|
klog.V(1).Infof("Found an IP %s which is no longer needed so cleaning up", addr.IP.String())
|
|
err := nsc.ln.ipAddrDel(dummyVipInterface, addr.IP.String())
|
|
if err != nil {
|
|
klog.Errorf("Failed to delete stale IP %s due to: %s",
|
|
addr.IP.String(), err.Error())
|
|
continue
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (nsc *NetworkServicesController) cleanupStaleIPVSConfig(activeServiceEndpointMap map[string][]string) error {
|
|
ipvsSvcs, err := nsc.ln.ipvsGetServices()
|
|
if err != nil {
|
|
return errors.New("failed get list of IPVS services due to: " + err.Error())
|
|
}
|
|
|
|
// cleanup stale ipvs service and servers
|
|
klog.V(1).Info("Cleaning up if any, old ipvs service and servers which are no longer needed")
|
|
|
|
var protocol string
|
|
for _, ipvsSvc := range ipvsSvcs {
|
|
// Note that this isn't all that safe of an assumption because FWMark services have a completely different
|
|
// protocol. So do SCTP services. However, we don't deal with SCTP in kube-router and FWMark is handled below.
|
|
protocol = convertSysCallProtoToSvcProto(ipvsSvc.Protocol)
|
|
// FWMark services by definition don't have a protocol, so we exclude those from the conditional so that they
|
|
// can be cleaned up correctly.
|
|
if protocol == noneProtocol && ipvsSvc.FWMark == 0 {
|
|
klog.Warningf("failed to convert protocol %d to a valid IPVS protocol for service: %s skipping",
|
|
ipvsSvc.Protocol, ipvsSvc.Address.String())
|
|
continue
|
|
}
|
|
|
|
var key string
|
|
switch {
|
|
case ipvsSvc.Address != nil:
|
|
key = generateIPPortID(ipvsSvc.Address.String(), protocol, strconv.Itoa(int(ipvsSvc.Port)))
|
|
case ipvsSvc.FWMark != 0:
|
|
key = fmt.Sprint(ipvsSvc.FWMark)
|
|
default:
|
|
continue
|
|
}
|
|
|
|
endpointIDs, ok := activeServiceEndpointMap[key]
|
|
// Only delete the service if it's not there anymore to prevent flapping
|
|
// old: if !ok || len(endpointIDs) == 0 {
|
|
if !ok {
|
|
excluded := false
|
|
for _, excludedCidr := range nsc.excludedCidrs {
|
|
if excludedCidr.Contains(ipvsSvc.Address) {
|
|
excluded = true
|
|
break
|
|
}
|
|
}
|
|
|
|
if excluded {
|
|
klog.V(1).Infof("Ignoring deletion of an IPVS service %s in an excluded cidr",
|
|
ipvsServiceString(ipvsSvc))
|
|
continue
|
|
}
|
|
|
|
klog.V(1).Infof("Found a IPVS service %s which is no longer needed so cleaning up",
|
|
ipvsServiceString(ipvsSvc))
|
|
if ipvsSvc.FWMark != 0 {
|
|
_, _, _, err = nsc.lookupServiceByFWMark(ipvsSvc.FWMark)
|
|
if err != nil {
|
|
klog.V(1).Infof("no FW mark found for service, nothing to cleanup: %v", err)
|
|
} else if err = nsc.cleanupDSRService(ipvsSvc.FWMark); err != nil {
|
|
klog.Errorf("failed to cleanup DSR service: %v", err)
|
|
}
|
|
}
|
|
err = nsc.ln.ipvsDelService(ipvsSvc)
|
|
if err != nil {
|
|
klog.Errorf("Failed to delete stale IPVS service %s due to: %s",
|
|
ipvsServiceString(ipvsSvc), err.Error())
|
|
continue
|
|
}
|
|
} else {
|
|
dsts, err := nsc.ln.ipvsGetDestinations(ipvsSvc)
|
|
if err != nil {
|
|
klog.Errorf("Failed to get list of servers from ipvs service")
|
|
}
|
|
for _, dst := range dsts {
|
|
validEp := false
|
|
for _, epID := range endpointIDs {
|
|
if epID == generateEndpointID(dst.Address.String(), strconv.Itoa(int(dst.Port))) {
|
|
validEp = true
|
|
break
|
|
}
|
|
}
|
|
if !validEp {
|
|
klog.V(1).Infof("Found a destination %s in service %s which is no longer needed so "+
|
|
"cleaning up", ipvsDestinationString(dst), ipvsServiceString(ipvsSvc))
|
|
err = nsc.ipvsDeleteDestination(ipvsSvc, dst)
|
|
if err != nil {
|
|
klog.Errorf("Failed to delete destination %s from ipvs service %s",
|
|
ipvsDestinationString(dst), ipvsServiceString(ipvsSvc))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// cleanupDSRService takes an FW mark was its only input and uses that to lookup the service and then remove DSR
|
|
// specific pieces of that service that may be left-over from the service provisioning.
|
|
func (nsc *NetworkServicesController) cleanupDSRService(fwMark uint32) error {
|
|
ipAddress, proto, port, err := nsc.lookupServiceByFWMark(fwMark)
|
|
if err != nil {
|
|
return fmt.Errorf("no service was found for FW mark: %d, service may not be all the way cleaned up: %v",
|
|
fwMark, err)
|
|
}
|
|
|
|
// cleanup mangle rules
|
|
klog.V(2).Infof("service %s:%s:%d was found, continuing with DSR service cleanup", ipAddress, proto, port)
|
|
mangleTableRulesDump := bytes.Buffer{}
|
|
var mangleTableRules []string
|
|
if err := utils.SaveInto("mangle", &mangleTableRulesDump); err != nil {
|
|
klog.Errorf("Failed to run iptables-save: %s" + err.Error())
|
|
} else {
|
|
mangleTableRules = strings.Split(mangleTableRulesDump.String(), "\n")
|
|
}
|
|
|
|
// All of the iptables-save output here prints FW marks in hexadecimal, if we are doing string searching, our search
|
|
// input needs to be in hex also
|
|
fwMarkStr := strconv.FormatInt(int64(fwMark), 16)
|
|
for _, mangleTableRule := range mangleTableRules {
|
|
if strings.Contains(mangleTableRule, ipAddress) && strings.Contains(mangleTableRule, fwMarkStr) {
|
|
klog.V(2).Infof("found mangle rule to cleanup: %s", mangleTableRule)
|
|
|
|
// When we cleanup the iptables rule, we need to pass FW mark as an int string rather than a hex string
|
|
err = nsc.ln.cleanupMangleTableRule(ipAddress, proto, strconv.Itoa(port), strconv.Itoa(int(fwMark)),
|
|
nsc.dsrTCPMSS)
|
|
if err != nil {
|
|
klog.Errorf("failed to verify and cleanup any mangle table rule to FORWARD the traffic "+
|
|
"to external IP due to: %v", err)
|
|
continue
|
|
} else {
|
|
// cleanupMangleTableRule will clean all rules in the table, so there is no need to continue looping
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
// cleanup the fwMarkMap to ensure that we don't accidentally build state
|
|
delete(nsc.fwMarkMap, fwMark)
|
|
return nil
|
|
}
|
|
|
|
func (nsc *NetworkServicesController) cleanupStaleMetrics(activeServiceEndpointMap map[string][]string) {
|
|
for k, v := range nsc.metricsMap {
|
|
if _, ok := activeServiceEndpointMap[k]; ok {
|
|
continue
|
|
}
|
|
|
|
metrics.ServiceBpsIn.DeleteLabelValues(v...)
|
|
metrics.ServiceBpsOut.DeleteLabelValues(v...)
|
|
metrics.ServiceBytesIn.DeleteLabelValues(v...)
|
|
metrics.ServiceBytesOut.DeleteLabelValues(v...)
|
|
metrics.ServiceCPS.DeleteLabelValues(v...)
|
|
metrics.ServicePacketsIn.DeleteLabelValues(v...)
|
|
metrics.ServicePacketsOut.DeleteLabelValues(v...)
|
|
metrics.ServicePpsIn.DeleteLabelValues(v...)
|
|
metrics.ServicePpsOut.DeleteLabelValues(v...)
|
|
metrics.ServiceTotalConn.DeleteLabelValues(v...)
|
|
metrics.ControllerIpvsServices.Dec()
|
|
delete(nsc.metricsMap, k)
|
|
}
|
|
}
|