mirrors/kube-router
1
0
Fork 0
mirror of https://github.com/cloudnativelabs/kube-router.git synced 2025-10-18 13:21:19 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #958 from coufalja/random-all

Browse Source 
Add --random-fully to MASQ iptables rules to mitigate conntrack issues
This commit is contained in:
Aaron U'Ren 2020-08-04 16:49:06 -05:00 committed by GitHub
commit e35dc9d61e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 240 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Gopkg.lock generated
View File

@ -98,12 +98,12 @@
version = "v0.5.2"
[[projects]]
digest = "1:df4af3a8e15cd72b068b8d898598001935edce4eac4eb5a3d7fd906036123ff4"
digest = "1:2debc00eaf8ffdbfb0570a791a84e64a3cfc146f5e61b54d4abd65c2de364552"
name = "github.com/coreos/go-iptables"
packages = ["iptables"]
pruneopts = "UT"
revision = "259c8e6a4275d497442c721fa52204d7a58bde8b"
version = "v0.2.0"
revision = "f901d6c2a4f2a4df092b98c33366dfba1f93d7a0"
version = "v0.4.5"
[[projects]]
digest = "1:a2c1d0e43bd3baaa071d1b9ed72c27d78169b2b269f71c105ac4ba34b1be4a39"
@ -1135,6 +1135,7 @@
"k8s.io/api/core/v1",
"k8s.io/api/extensions/v1beta1",
"k8s.io/api/networking/v1",
"k8s.io/apimachinery/pkg/api/resource",
"k8s.io/apimachinery/pkg/apis/meta/v1",
"k8s.io/apimachinery/pkg/labels",
"k8s.io/apimachinery/pkg/util/intstr",

2
Gopkg.toml
View File

@ -37,7 +37,7 @@ required = ["github.com/osrg/gobgp/gobgp"]
[[constraint]]
name = "github.com/coreos/go-iptables"
version = "0.2.0"
version = "0.4.1"
[[constraint]]
branch = "master"

17
pkg/controllers/proxy/network_services_controller.go
View File

@ -1225,6 +1225,9 @@ func (nsc *NetworkServicesController) ensureMasqueradeIptablesRule() error {
return errors.New("Failed to initialize iptables executor" + err.Error())
}
var args = []string{"-m", "ipvs", "--ipvs", "--vdir", "ORIGINAL", "--vmethod", "MASQ", "-m", "comment", "--comment", "", "-j", "SNAT", "--to-source", nsc.nodeIP.String()}
if iptablesCmdHandler.HasRandomFully() {
args = append(args, "--random-fully")
}
if nsc.masqueradeAll {
err = iptablesCmdHandler.AppendUnique("nat", "POSTROUTING", args...)
if err != nil {
@ -1248,6 +1251,10 @@ func (nsc *NetworkServicesController) ensureMasqueradeIptablesRule() error {
//TODO: ipset should be used for destination podCidr(s) match after multiple podCidr(s) per node get supported
args = []string{"-m", "ipvs", "--ipvs", "--vdir", "ORIGINAL", "--vmethod", "MASQ", "-m", "comment", "--comment", "",
"!", "-s", nsc.podCidr, "!", "-d", nsc.podCidr, "-j", "SNAT", "--to-source", nsc.nodeIP.String()}
if iptablesCmdHandler.HasRandomFully() {
args = append(args, "--random-fully")
}
err = iptablesCmdHandler.AppendUnique("nat", "POSTROUTING", args...)
if err != nil {
return errors.New("Failed to run iptables command" + err.Error())
@ -1269,6 +1276,16 @@ func (nsc *NetworkServicesController) deleteBadMasqueradeIptablesRules() error {
{"-m", "ipvs", "--ipvs", "--vdir", "ORIGINAL", "--vmethod", "MASQ", "-m", "comment", "--comment", "", "!", "-s", nsc.podCidr, "!", "-d", nsc.podCidr, "-j", "MASQUERADE"},
}
// If random fully is supported remove the original rules as well
if iptablesCmdHandler.HasRandomFully() {
argsBad = append(argsBad, []string{"-m", "ipvs", "--ipvs", "--vdir", "ORIGINAL", "--vmethod", "MASQ", "-m", "comment", "--comment", "", "-j", "SNAT", "--to-source", nsc.nodeIP.String()})
if len(nsc.podCidr) > 0 {
argsBad = append(argsBad, []string{"-m", "ipvs", "--ipvs", "--vdir", "ORIGINAL", "--vmethod", "MASQ", "-m", "comment", "--comment", "",
"!", "-s", nsc.podCidr, "!", "-d", nsc.podCidr, "-j", "SNAT", "--to-source", nsc.nodeIP.String()})
}
}
for _, args := range argsBad {
exists, err := iptablesCmdHandler.Exists("nat", "POSTROUTING", args...)
if err != nil {

18
pkg/controllers/routing/pod_egress.go
View File

@ -36,6 +36,10 @@ func (nrc *NetworkRoutingController) createPodEgressRule() error {
if nrc.isIpv6 {
podEgressArgs = podEgressArgs6
}
if iptablesCmdHandler.HasRandomFully() {
podEgressArgs = append(podEgressArgs, "--random-fully")
}
err = iptablesCmdHandler.AppendUnique("nat", "POSTROUTING", podEgressArgs...)
if err != nil {
return errors.New("Failed to add iptables rule to masquerade outbound traffic from pods: " +
@ -57,6 +61,10 @@ func (nrc *NetworkRoutingController) deletePodEgressRule() error {
if nrc.isIpv6 {
podEgressArgs = podEgressArgs6
}
if iptablesCmdHandler.HasRandomFully() {
podEgressArgs = append(podEgressArgs, "--random-fully")
}
exists, err := iptablesCmdHandler.Exists("nat", "POSTROUTING", podEgressArgs...)
if err != nil {
return errors.New("Failed to lookup iptables rule to masquerade outbound traffic from pods: " + err.Error())
@ -83,6 +91,16 @@ func (nrc *NetworkRoutingController) deleteBadPodEgressRules() error {
if nrc.isIpv6 {
podEgressArgsBad = podEgressArgsBad6
}
// If random fully is supported remove the original rule as well
if iptablesCmdHandler.HasRandomFully() {
if !nrc.isIpv6 {
podEgressArgsBad = append(podEgressArgsBad, podEgressArgs4)
} else {
podEgressArgsBad = append(podEgressArgsBad, podEgressArgs6)
}
}
for _, args := range podEgressArgsBad {
exists, err := iptablesCmdHandler.Exists("nat", "POSTROUTING", args...)
if err != nil {

5
vendor/github.com/coreos/go-iptables/NOTICE generated vendored Normal file
View File

@ -0,0 +1,5 @@
CoreOS Project
Copyright 2018 CoreOS, Inc
This product includes software developed at CoreOS, Inc.
(http://www.coreos.com/).

210
vendor/github.com/coreos/go-iptables/iptables/iptables.go generated vendored
View File

@ -31,9 +31,14 @@ type Error struct {
exec.ExitError
cmd exec.Cmd
msg string
proto Protocol
exitStatus *int //for overriding
}
func (e *Error) ExitStatus() int {
if e.exitStatus != nil {
return *e.exitStatus
}
return e.Sys().(syscall.WaitStatus).ExitStatus()
}
@ -41,6 +46,17 @@ func (e *Error) Error() string {
return fmt.Sprintf("running %v: exit status %v: %v", e.cmd.Args, e.ExitStatus(), e.msg)
}
// IsNotExist returns true if the error is due to the chain or rule not existing
func (e *Error) IsNotExist() bool {
if e.ExitStatus() != 1 {
return false
}
cmdIptables := getIptablesCommand(e.proto)
msgNoRuleExist := fmt.Sprintf("%s: Bad rule (does a matching rule exist in that chain?).\n", cmdIptables)
msgNoChainExist := fmt.Sprintf("%s: No chain/target/match by that name.\n", cmdIptables)
return strings.Contains(e.msg, msgNoRuleExist) || strings.Contains(e.msg, msgNoChainExist)
}
// Protocol to differentiate between IPv4 and IPv6
type Protocol byte
@ -54,6 +70,25 @@ type IPTables struct {
proto Protocol
hasCheck bool
hasWait bool
hasRandomFully bool
v1 int
v2 int
v3 int
mode string // the underlying iptables operating mode, e.g. nf_tables
}
// Stat represents a structured statistic entry.
type Stat struct {
Packets uint64 `json:"pkts"`
Bytes uint64 `json:"bytes"`
Target string `json:"target"`
Protocol string `json:"prot"`
Opt string `json:"opt"`
Input string `json:"in"`
Output string `json:"out"`
Source *net.IPNet `json:"source"`
Destination *net.IPNet `json:"destination"`
Options string `json:"options"`
}
// New creates a new IPTables.
@ -69,15 +104,27 @@ func NewWithProtocol(proto Protocol) (*IPTables, error) {
if err != nil {
return nil, err
}
checkPresent, waitPresent, err := getIptablesCommandSupport(path)
vstring, err := getIptablesVersionString(path)
if err != nil {
return nil, fmt.Errorf("error checking iptables version: %v", err)
return nil, fmt.Errorf("could not get iptables version: %v", err)
}
v1, v2, v3, mode, err := extractIptablesVersion(vstring)
if err != nil {
return nil, fmt.Errorf("failed to extract iptables version from [%s]: %v", vstring, err)
}
checkPresent, waitPresent, randomFullyPresent := getIptablesCommandSupport(v1, v2, v3)
ipt := IPTables{
path: path,
proto: proto,
hasCheck: checkPresent,
hasWait: waitPresent,
hasRandomFully: randomFullyPresent,
v1: v1,
v2: v2,
v3: v3,
mode: mode,
}
return &ipt, nil
}
@ -241,6 +288,63 @@ func (ipt *IPTables) Stats(table, chain string) ([][]string, error) {
return rows, nil
}
// ParseStat parses a single statistic row into a Stat struct. The input should
// be a string slice that is returned from calling the Stat method.
func (ipt *IPTables) ParseStat(stat []string) (parsed Stat, err error) {
// For forward-compatibility, expect at least 10 fields in the stat
if len(stat) < 10 {
return parsed, fmt.Errorf("stat contained fewer fields than expected")
}
// Convert the fields that are not plain strings
parsed.Packets, err = strconv.ParseUint(stat[0], 0, 64)
if err != nil {
return parsed, fmt.Errorf(err.Error(), "could not parse packets")
}
parsed.Bytes, err = strconv.ParseUint(stat[1], 0, 64)
if err != nil {
return parsed, fmt.Errorf(err.Error(), "could not parse bytes")
}
_, parsed.Source, err = net.ParseCIDR(stat[7])
if err != nil {
return parsed, fmt.Errorf(err.Error(), "could not parse source")
}
_, parsed.Destination, err = net.ParseCIDR(stat[8])
if err != nil {
return parsed, fmt.Errorf(err.Error(), "could not parse destination")
}
// Put the fields that are strings
parsed.Target = stat[2]
parsed.Protocol = stat[3]
parsed.Opt = stat[4]
parsed.Input = stat[5]
parsed.Output = stat[6]
parsed.Options = stat[9]
return parsed, nil
}
// StructuredStats returns statistics as structured data which may be further
// parsed and marshaled.
func (ipt *IPTables) StructuredStats(table, chain string) ([]Stat, error) {
rawStats, err := ipt.Stats(table, chain)
if err != nil {
return nil, err
}
structStats := []Stat{}
for _, rawStat := range rawStats {
stat, err := ipt.ParseStat(rawStat)
if err != nil {
return nil, err
}
structStats = append(structStats, stat)
}
return structStats, nil
}
func (ipt *IPTables) executeList(args []string) ([]string, error) {
var stdout bytes.Buffer
if err := ipt.runWithOutput(args, &stdout); err != nil {
@ -248,10 +352,16 @@ func (ipt *IPTables) executeList(args []string) ([]string, error) {
}
rules := strings.Split(stdout.String(), "\n")
// strip trailing newline
if len(rules) > 0 && rules[len(rules)-1] == "" {
rules = rules[:len(rules)-1]
}
for i, rule := range rules {
rules[i] = filterRuleOutput(rule)
}
return rules, nil
}
@ -261,6 +371,8 @@ func (ipt *IPTables) NewChain(table, chain string) error {
return ipt.run("-t", table, "-N", chain)
}
const existsErr = 1
// ClearChain flushed (deletes all rules) in the specified table/chain.
// If the chain does not exist, a new one will be created
func (ipt *IPTables) ClearChain(table, chain string) error {
@ -270,7 +382,7 @@ func (ipt *IPTables) ClearChain(table, chain string) error {
switch {
case err == nil:
return nil
case eok && eerr.ExitStatus() == 1:
case eok && eerr.ExitStatus() == existsErr:
// chain already exists. Flush (clear) it.
return ipt.run("-t", table, "-F", chain)
default:
@ -289,6 +401,21 @@ func (ipt *IPTables) DeleteChain(table, chain string) error {
return ipt.run("-t", table, "-X", chain)
}
// ChangePolicy changes policy on chain to target
func (ipt *IPTables) ChangePolicy(table, chain, target string) error {
return ipt.run("-t", table, "-P", chain, target)
}
// Check if the underlying iptables command supports the --random-fully flag
func (ipt *IPTables) HasRandomFully() bool {
return ipt.hasRandomFully
}
// Return version components of the underlying iptables command
func (ipt *IPTables) GetIptablesVersion() (int, int, int) {
return ipt.v1, ipt.v2, ipt.v3
}
// run runs an iptables command with the given arguments, ignoring
// any stdout output
func (ipt *IPTables) run(args ...string) error {
@ -308,6 +435,7 @@ func (ipt *IPTables) runWithOutput(args []string, stdout io.Writer) error {
}
ul, err := fmu.tryLock()
if err != nil {
syscall.Close(fmu.fd)
return err
}
defer ul.Unlock()
@ -324,7 +452,7 @@ func (ipt *IPTables) runWithOutput(args []string, stdout io.Writer) error {
if err := cmd.Run(); err != nil {
switch e := err.(type) {
case *exec.ExitError:
return &Error{*e, cmd, stderr.String()}
return &Error{*e, cmd, stderr.String(), ipt.proto, nil}
default:
return err
}
@ -343,45 +471,40 @@ func getIptablesCommand(proto Protocol) string {
}
// Checks if iptables has the "-C" and "--wait" flag
func getIptablesCommandSupport(path string) (bool, bool, error) {
vstring, err := getIptablesVersionString(path)
if err != nil {
return false, false, err
func getIptablesCommandSupport(v1 int, v2 int, v3 int) (bool, bool, bool) {
return iptablesHasCheckCommand(v1, v2, v3), iptablesHasWaitCommand(v1, v2, v3), iptablesHasRandomFully(v1, v2, v3)
}
v1, v2, v3, err := extractIptablesVersion(vstring)
if err != nil {
return false, false, err
}
return iptablesHasCheckCommand(v1, v2, v3), iptablesHasWaitCommand(v1, v2, v3), nil
}
// getIptablesVersion returns the first three components of the iptables version.
// e.g. "iptables v1.3.66" would return (1, 3, 66, nil)
func extractIptablesVersion(str string) (int, int, int, error) {
versionMatcher := regexp.MustCompile("v([0-9]+)\\.([0-9]+)\\.([0-9]+)")
// getIptablesVersion returns the first three components of the iptables version
// and the operating mode (e.g. nf_tables or legacy)
// e.g. "iptables v1.3.66" would return (1, 3, 66, legacy, nil)
func extractIptablesVersion(str string) (int, int, int, string, error) {
versionMatcher := regexp.MustCompile(`v([0-9]+)\.([0-9]+)\.([0-9]+)(?:\s+\((\w+))?`)
result := versionMatcher.FindStringSubmatch(str)
if result == nil {
return 0, 0, 0, fmt.Errorf("no iptables version found in string: %s", str)
return 0, 0, 0, "", fmt.Errorf("no iptables version found in string: %s", str)
}
v1, err := strconv.Atoi(result[1])
if err != nil {
return 0, 0, 0, err
return 0, 0, 0, "", err
}
v2, err := strconv.Atoi(result[2])
if err != nil {
return 0, 0, 0, err
return 0, 0, 0, "", err
}
v3, err := strconv.Atoi(result[3])
if err != nil {
return 0, 0, 0, err
return 0, 0, 0, "", err
}
return v1, v2, v3, nil
mode := "legacy"
if result[4] != "" {
mode = result[4]
}
return v1, v2, v3, mode, nil
}
// Runs "iptables --version" to get the version string
@ -424,6 +547,20 @@ func iptablesHasWaitCommand(v1 int, v2 int, v3 int) bool {
return false
}
// Checks if an iptables version is after 1.6.2, when --random-fully was added
func iptablesHasRandomFully(v1 int, v2 int, v3 int) bool {
if v1 > 1 {
return true
}
if v1 == 1 && v2 > 6 {
return true
}
if v1 == 1 && v2 == 6 && v3 >= 2 {
return true
}
return false
}
// Checks if a rule specification exists for a table
func (ipt *IPTables) existsForOldIptables(table, chain string, rulespec []string) (bool, error) {
rs := strings.Join(append([]string{"-A", chain}, rulespec...), " ")
@ -435,3 +572,26 @@ func (ipt *IPTables) existsForOldIptables(table, chain string, rulespec []string
}
return strings.Contains(stdout.String(), rs), nil
}
// counterRegex is the regex used to detect nftables counter format
var counterRegex = regexp.MustCompile(`^\[([0-9]+):([0-9]+)\] `)
// filterRuleOutput works around some inconsistencies in output.
// For example, when iptables is in legacy vs. nftables mode, it produces
// different results.
func filterRuleOutput(rule string) string {
out := rule
// work around an output difference in nftables mode where counters
// are output in iptables-save format, rather than iptables -S format
// The string begins with "[0:0]"
//
// Fixes #49
if groups := counterRegex.FindStringSubmatch(out); groups != nil {
// drop the brackets
out = out[len(groups[0]):]
out = fmt.Sprintf("%s -c %s %s", out, groups[1], groups[2])
}
return out
}