From 077ff86bcc558364990e0ae2db12ceb8d27d13c5 Mon Sep 17 00:00:00 2001 From: Lars Ekman Date: Wed, 10 Oct 2018 07:32:59 +0200 Subject: [PATCH] Ipv6; BGP peering (#545) * Use ip6tables for ipv6 and handle ipv6 for egress rules * Make the temp ipset's fit into 31 characters This should be improved. Some hash string should be used for temp names. --- .../routing/network_routes_controller.go | 56 +++++++++++-------- pkg/controllers/routing/pod_egress.go | 37 ++++++++---- pkg/controllers/routing/utils.go | 2 +- pkg/utils/ipset.go | 3 +- 4 files changed, 63 insertions(+), 35 deletions(-) diff --git a/pkg/controllers/routing/network_routes_controller.go b/pkg/controllers/routing/network_routes_controller.go index 5f8a6311..387a4bd0 100644 --- a/pkg/controllers/routing/network_routes_controller.go +++ b/pkg/controllers/routing/network_routes_controller.go @@ -60,6 +60,7 @@ type NetworkRoutingController struct { nodeName string nodeSubnet net.IPNet nodeInterface string + isIpv6 bool activeNodes map[string]bool mu sync.Mutex clientset kubernetes.Interface @@ -151,7 +152,7 @@ func (nrc *NetworkRoutingController) Run(healthChan chan<- *healthcheck.Controll } glog.V(1).Info("Performing cleanup of depreciated rules/ipsets (if needed).") - err = deleteBadPodEgressRules() + err = nrc.deleteBadPodEgressRules() if err != nil { glog.Errorf("Error cleaning up old/bad Pod egress rules: %s", err.Error()) } @@ -160,14 +161,14 @@ func (nrc *NetworkRoutingController) Run(healthChan chan<- *healthcheck.Controll if nrc.enablePodEgress { glog.V(1).Infoln("Enabling Pod egress.") - err = createPodEgressRule() + err = nrc.createPodEgressRule() if err != nil { glog.Errorf("Error enabling Pod egress: %s", err.Error()) } } else { glog.V(1).Infoln("Disabling Pod egress.") - err = deletePodEgressRule() + err = nrc.deletePodEgressRule() if err != nil { glog.Warningf("Error cleaning up Pod Egress related networking: %s", err) } @@ -458,18 +459,18 @@ func (nrc *NetworkRoutingController) injectRoute(path *table.Path) error { // Cleanup performs the cleanup of configurations done func (nrc *NetworkRoutingController) Cleanup() { // Pod egress cleanup - err := deletePodEgressRule() + err := nrc.deletePodEgressRule() if err != nil { glog.Warningf("Error deleting Pod egress iptable rule: %s", err.Error()) } - err = deleteBadPodEgressRules() + err = nrc.deleteBadPodEgressRules() if err != nil { glog.Warningf("Error deleting Pod egress iptable rule: %s", err.Error()) } // delete all ipsets created by kube-router - ipset, err := utils.NewIPSet(false) + ipset, err := utils.NewIPSet(nrc.isIpv6) if err != nil { glog.Errorf("Failed to clean up ipsets: " + err.Error()) } @@ -535,12 +536,20 @@ func (nrc *NetworkRoutingController) syncNodeIPSets() error { return nil } +func (nrc *NetworkRoutingController) newIptablesCmdHandler() (*iptables.IPTables, error) { + if nrc.isIpv6 { + return iptables.NewWithProtocol(iptables.ProtocolIPv6) + } else { + return iptables.NewWithProtocol(iptables.ProtocolIPv4) + } +} + // ensure there is rule in filter table and FORWARD chain to permit in/out traffic from pods // this rules will be appended so that any iptable rules for network policies will take // precedence func (nrc *NetworkRoutingController) enableForwarding() error { - iptablesCmdHandler, err := iptables.New() + iptablesCmdHandler, _ := nrc.newIptablesCmdHandler() comment := "allow outbound traffic from pods" args := []string{"-m", "comment", "--comment", comment, "-i", "kube-bridge", "-j", "ACCEPT"} @@ -661,7 +670,7 @@ func (nrc *NetworkRoutingController) startBgpServer() error { } if ipv6IsEnabled() { - localAddressList = append(localAddressList, "::") + localAddressList = append(localAddressList, "::1") } global := &config.Global{ @@ -795,6 +804,21 @@ func NewNetworkRoutingController(clientset kubernetes.Interface, nrc.disableSrcDstCheck = kubeRouterConfig.DisableSrcDstCheck nrc.initSrcDstCheckDone = false + nrc.hostnameOverride = kubeRouterConfig.HostnameOverride + node, err := utils.GetNodeObject(clientset, nrc.hostnameOverride) + if err != nil { + return nil, errors.New("Failed getting node object from API server: " + err.Error()) + } + + nrc.nodeName = node.Name + + nodeIP, err := utils.GetNodeIP(node) + if err != nil { + return nil, errors.New("Failed getting IP address from node object: " + err.Error()) + } + nrc.nodeIP = nodeIP + nrc.isIpv6 = nodeIP.To4() == nil + // lets start with assumption we hace necessary IAM creds to access EC2 api nrc.ec2IamAuthorized = true @@ -808,7 +832,7 @@ func NewNetworkRoutingController(clientset kubernetes.Interface, } } - nrc.ipSetHandler, err = utils.NewIPSet(false) + nrc.ipSetHandler, err = utils.NewIPSet(nrc.isIpv6) if err != nil { return nil, err } @@ -872,20 +896,6 @@ func NewNetworkRoutingController(clientset kubernetes.Interface, return nil, fmt.Errorf("Error processing Global Peer Router configs: %s", err) } - nrc.hostnameOverride = kubeRouterConfig.HostnameOverride - node, err := utils.GetNodeObject(clientset, nrc.hostnameOverride) - if err != nil { - return nil, errors.New("Failed getting node object from API server: " + err.Error()) - } - - nrc.nodeName = node.Name - - nodeIP, err := utils.GetNodeIP(node) - if err != nil { - return nil, errors.New("Failed getting IP address from node object: " + err.Error()) - } - nrc.nodeIP = nodeIP - nrc.nodeSubnet, nrc.nodeInterface, err = getNodeSubnet(nodeIP) if err != nil { return nil, errors.New("Failed find the subnet of the node IP and interface on" + diff --git a/pkg/controllers/routing/pod_egress.go b/pkg/controllers/routing/pod_egress.go index b09add54..22e41b6a 100644 --- a/pkg/controllers/routing/pod_egress.go +++ b/pkg/controllers/routing/pod_egress.go @@ -4,28 +4,38 @@ import ( "errors" "fmt" - "github.com/coreos/go-iptables/iptables" "github.com/golang/glog" ) // set up MASQUERADE rule so that egress traffic from the pods gets masquraded to node's IP var ( - podEgressArgs = []string{"-m", "set", "--match-set", podSubnetsIPSetName, "src", + podEgressArgs4 = []string{"-m", "set", "--match-set", podSubnetsIPSetName, "src", "-m", "set", "!", "--match-set", podSubnetsIPSetName, "dst", "-m", "set", "!", "--match-set", nodeAddrsIPSetName, "dst", "-j", "MASQUERADE"} - podEgressArgsBad = [][]string{{"-m", "set", "--match-set", podSubnetsIPSetName, "src", + podEgressArgs6 = []string{"-m", "set", "--match-set", "inet6:" + podSubnetsIPSetName, "src", + "-m", "set", "!", "--match-set", "inet6:" + podSubnetsIPSetName, "dst", + "-m", "set", "!", "--match-set", "inet6:" + nodeAddrsIPSetName, "dst", + "-j", "MASQUERADE"} + podEgressArgsBad4 = [][]string{{"-m", "set", "--match-set", podSubnetsIPSetName, "src", "-m", "set", "!", "--match-set", podSubnetsIPSetName, "dst", "-j", "MASQUERADE"}} + podEgressArgsBad6 = [][]string{{"-m", "set", "--match-set", "inet6:" + podSubnetsIPSetName, "src", + "-m", "set", "!", "--match-set", "inet6:" + podSubnetsIPSetName, "dst", + "-j", "MASQUERADE"}} ) -func createPodEgressRule() error { - iptablesCmdHandler, err := iptables.New() +func (nrc *NetworkRoutingController) createPodEgressRule() error { + iptablesCmdHandler, err := nrc.newIptablesCmdHandler() if err != nil { return errors.New("Failed create iptables handler:" + err.Error()) } + podEgressArgs := podEgressArgs4 + if nrc.isIpv6 { + podEgressArgs = podEgressArgs6 + } err = iptablesCmdHandler.AppendUnique("nat", "POSTROUTING", podEgressArgs...) if err != nil { return errors.New("Failed to add iptable rule to masqurade outbound traffic from pods: " + @@ -37,12 +47,16 @@ func createPodEgressRule() error { return nil } -func deletePodEgressRule() error { - iptablesCmdHandler, err := iptables.New() +func (nrc *NetworkRoutingController) deletePodEgressRule() error { + iptablesCmdHandler, err := nrc.newIptablesCmdHandler() if err != nil { return errors.New("Failed create iptables handler:" + err.Error()) } + podEgressArgs := podEgressArgs4 + if nrc.isIpv6 { + podEgressArgs = podEgressArgs6 + } exists, err := iptablesCmdHandler.Exists("nat", "POSTROUTING", podEgressArgs...) if err != nil { return errors.New("Failed to lookup iptable rule to masqurade outbound traffic from pods: " + err.Error()) @@ -60,12 +74,15 @@ func deletePodEgressRule() error { return nil } -func deleteBadPodEgressRules() error { - iptablesCmdHandler, err := iptables.New() +func (nrc *NetworkRoutingController) deleteBadPodEgressRules() error { + iptablesCmdHandler, err := nrc.newIptablesCmdHandler() if err != nil { return errors.New("Failed create iptables handler:" + err.Error()) } - + podEgressArgsBad := podEgressArgsBad4 + if nrc.isIpv6 { + podEgressArgsBad = podEgressArgsBad6 + } for _, args := range podEgressArgsBad { exists, err := iptablesCmdHandler.Exists("nat", "POSTROUTING", args...) if err != nil { diff --git a/pkg/controllers/routing/utils.go b/pkg/controllers/routing/utils.go index afb82552..2bf53d25 100644 --- a/pkg/controllers/routing/utils.go +++ b/pkg/controllers/routing/utils.go @@ -98,7 +98,7 @@ func getNodeSubnet(nodeIp net.IP) (net.IPNet, string, error) { return net.IPNet{}, "", errors.New("Failed to get list of links") } for _, link := range links { - addresses, err := netlink.AddrList(link, netlink.FAMILY_V4) + addresses, err := netlink.AddrList(link, netlink.FAMILY_ALL) if err != nil { return net.IPNet{}, "", errors.New("Failed to get list of addr") } diff --git a/pkg/utils/ipset.go b/pkg/utils/ipset.go index 36d60082..dbf8efce 100644 --- a/pkg/utils/ipset.go +++ b/pkg/utils/ipset.go @@ -440,7 +440,8 @@ func (set *Set) Swap(setTo *Set) error { // Refresh a Set with new entries. func (set *Set) Refresh(entries []string, extraOptions ...string) error { var err error - tempName := set.Name + "-temp" + // The set-name must be < 32 characters! + tempName := set.Name + "-" newSet := &Set{ Parent: set.Parent,