Merge pull request #958 from coufalja/random-all

Add --random-fully to MASQ iptables rules to mitigate conntrack issues

Gopkg.lock

@@ -98,12 +98,12 @@
   version = "v0.5.2"
 
 [[projects]]
-  digest = "1:df4af3a8e15cd72b068b8d898598001935edce4eac4eb5a3d7fd906036123ff4"
+  digest = "1:2debc00eaf8ffdbfb0570a791a84e64a3cfc146f5e61b54d4abd65c2de364552"
   name = "github.com/coreos/go-iptables"
   packages = ["iptables"]
   pruneopts = "UT"
-  revision = "259c8e6a4275d497442c721fa52204d7a58bde8b"
+  revision = "f901d6c2a4f2a4df092b98c33366dfba1f93d7a0"
-  version = "v0.2.0"
+  version = "v0.4.5"
 
 [[projects]]
   digest = "1:a2c1d0e43bd3baaa071d1b9ed72c27d78169b2b269f71c105ac4ba34b1be4a39"
@@ -1135,6 +1135,7 @@
     "k8s.io/api/core/v1",
     "k8s.io/api/extensions/v1beta1",
     "k8s.io/api/networking/v1",
+    "k8s.io/apimachinery/pkg/api/resource",
     "k8s.io/apimachinery/pkg/apis/meta/v1",
     "k8s.io/apimachinery/pkg/labels",
     "k8s.io/apimachinery/pkg/util/intstr",

Gopkg.toml

@@ -37,7 +37,7 @@ required = ["github.com/osrg/gobgp/gobgp"]
 
 [[constraint]]
   name = "github.com/coreos/go-iptables"
-  version = "0.2.0"
+  version = "0.4.1"
 
 [[constraint]]
   branch = "master"

pkg/controllers/proxy/network_services_controller.go

@@ -1225,6 +1225,9 @@ func (nsc *NetworkServicesController) ensureMasqueradeIptablesRule() error {
 		return errors.New("Failed to initialize iptables executor" + err.Error())
 	}
 	var args = []string{"-m", "ipvs", "--ipvs", "--vdir", "ORIGINAL", "--vmethod", "MASQ", "-m", "comment", "--comment", "", "-j", "SNAT", "--to-source", nsc.nodeIP.String()}
+	if iptablesCmdHandler.HasRandomFully() {
+		args = append(args, "--random-fully")
+	}
 	if nsc.masqueradeAll {
 		err = iptablesCmdHandler.AppendUnique("nat", "POSTROUTING", args...)
 		if err != nil {
@@ -1248,6 +1251,10 @@ func (nsc *NetworkServicesController) ensureMasqueradeIptablesRule() error {
 		//TODO: ipset should be used for destination podCidr(s) match after multiple podCidr(s) per node get supported
 		args = []string{"-m", "ipvs", "--ipvs", "--vdir", "ORIGINAL", "--vmethod", "MASQ", "-m", "comment", "--comment", "",
 			"!", "-s", nsc.podCidr, "!", "-d", nsc.podCidr, "-j", "SNAT", "--to-source", nsc.nodeIP.String()}
+		if iptablesCmdHandler.HasRandomFully() {
+			args = append(args, "--random-fully")
+		}
+
 		err = iptablesCmdHandler.AppendUnique("nat", "POSTROUTING", args...)
 		if err != nil {
 			return errors.New("Failed to run iptables command" + err.Error())
@@ -1269,6 +1276,16 @@ func (nsc *NetworkServicesController) deleteBadMasqueradeIptablesRules() error {
 		{"-m", "ipvs", "--ipvs", "--vdir", "ORIGINAL", "--vmethod", "MASQ", "-m", "comment", "--comment", "", "!", "-s", nsc.podCidr, "!", "-d", nsc.podCidr, "-j", "MASQUERADE"},
 	}
 
+	// If random fully is supported remove the original rules as well
+	if iptablesCmdHandler.HasRandomFully() {
+		argsBad = append(argsBad, []string{"-m", "ipvs", "--ipvs", "--vdir", "ORIGINAL", "--vmethod", "MASQ", "-m", "comment", "--comment", "", "-j", "SNAT", "--to-source", nsc.nodeIP.String()})
+
+		if len(nsc.podCidr) > 0 {
+			argsBad = append(argsBad, []string{"-m", "ipvs", "--ipvs", "--vdir", "ORIGINAL", "--vmethod", "MASQ", "-m", "comment", "--comment", "",
+				"!", "-s", nsc.podCidr, "!", "-d", nsc.podCidr, "-j", "SNAT", "--to-source", nsc.nodeIP.String()})
+		}
+	}
+
 	for _, args := range argsBad {
 		exists, err := iptablesCmdHandler.Exists("nat", "POSTROUTING", args...)
 		if err != nil {

pkg/controllers/routing/pod_egress.go

@@ -36,6 +36,10 @@ func (nrc *NetworkRoutingController) createPodEgressRule() error {
 	if nrc.isIpv6 {
 		podEgressArgs = podEgressArgs6
 	}
+	if iptablesCmdHandler.HasRandomFully() {
+		podEgressArgs = append(podEgressArgs, "--random-fully")
+	}
+
 	err = iptablesCmdHandler.AppendUnique("nat", "POSTROUTING", podEgressArgs...)
 	if err != nil {
 		return errors.New("Failed to add iptables rule to masquerade outbound traffic from pods: " +
@@ -57,6 +61,10 @@ func (nrc *NetworkRoutingController) deletePodEgressRule() error {
 	if nrc.isIpv6 {
 		podEgressArgs = podEgressArgs6
 	}
+	if iptablesCmdHandler.HasRandomFully() {
+		podEgressArgs = append(podEgressArgs, "--random-fully")
+	}
+
 	exists, err := iptablesCmdHandler.Exists("nat", "POSTROUTING", podEgressArgs...)
 	if err != nil {
 		return errors.New("Failed to lookup iptables rule to masquerade outbound traffic from pods: " + err.Error())
@@ -83,6 +91,16 @@ func (nrc *NetworkRoutingController) deleteBadPodEgressRules() error {
 	if nrc.isIpv6 {
 		podEgressArgsBad = podEgressArgsBad6
 	}
+
+	// If random fully is supported remove the original rule as well
+	if iptablesCmdHandler.HasRandomFully() {
+		if !nrc.isIpv6 {
+			podEgressArgsBad = append(podEgressArgsBad, podEgressArgs4)
+		} else {
+			podEgressArgsBad = append(podEgressArgsBad, podEgressArgs6)
+		}
+	}
+
 	for _, args := range podEgressArgsBad {
 		exists, err := iptablesCmdHandler.Exists("nat", "POSTROUTING", args...)
 		if err != nil {

vendor/github.com/coreos/go-iptables/NOTICE

@@ -0,0 +1,5 @@
+CoreOS Project
+Copyright 2018 CoreOS, Inc
+
+This product includes software developed at CoreOS, Inc.
+(http://www.coreos.com/).

vendor/github.com/coreos/go-iptables/iptables/iptables.go

@@ -29,11 +29,16 @@ import (
 // Adds the output of stderr to exec.ExitError
 type Error struct {
 	exec.ExitError
-	cmd exec.Cmd
+	cmd        exec.Cmd
-	msg string
+	msg        string
+	proto      Protocol
+	exitStatus *int //for overriding
 }
 
 func (e *Error) ExitStatus() int {
+	if e.exitStatus != nil {
+		return *e.exitStatus
+	}
 	return e.Sys().(syscall.WaitStatus).ExitStatus()
 }
 
@@ -41,6 +46,17 @@ func (e *Error) Error() string {
 	return fmt.Sprintf("running %v: exit status %v: %v", e.cmd.Args, e.ExitStatus(), e.msg)
 }
 
+// IsNotExist returns true if the error is due to the chain or rule not existing
+func (e *Error) IsNotExist() bool {
+	if e.ExitStatus() != 1 {
+		return false
+	}
+	cmdIptables := getIptablesCommand(e.proto)
+	msgNoRuleExist := fmt.Sprintf("%s: Bad rule (does a matching rule exist in that chain?).\n", cmdIptables)
+	msgNoChainExist := fmt.Sprintf("%s: No chain/target/match by that name.\n", cmdIptables)
+	return strings.Contains(e.msg, msgNoRuleExist) || strings.Contains(e.msg, msgNoChainExist)
+}
+
 // Protocol to differentiate between IPv4 and IPv6
 type Protocol byte
 
@@ -50,10 +66,29 @@ const (
 )
 
 type IPTables struct {
-	path     string
+	path           string
-	proto    Protocol
+	proto          Protocol
-	hasCheck bool
+	hasCheck       bool
-	hasWait  bool
+	hasWait        bool
+	hasRandomFully bool
+	v1             int
+	v2             int
+	v3             int
+	mode           string // the underlying iptables operating mode, e.g. nf_tables
+}
+
+// Stat represents a structured statistic entry.
+type Stat struct {
+	Packets     uint64     `json:"pkts"`
+	Bytes       uint64     `json:"bytes"`
+	Target      string     `json:"target"`
+	Protocol    string     `json:"prot"`
+	Opt         string     `json:"opt"`
+	Input       string     `json:"in"`
+	Output      string     `json:"out"`
+	Source      *net.IPNet `json:"source"`
+	Destination *net.IPNet `json:"destination"`
+	Options     string     `json:"options"`
 }
 
 // New creates a new IPTables.
@@ -69,15 +104,27 @@ func NewWithProtocol(proto Protocol) (*IPTables, error) {
 	if err != nil {
 		return nil, err
 	}
-	checkPresent, waitPresent, err := getIptablesCommandSupport(path)
+	vstring, err := getIptablesVersionString(path)
 	if err != nil {
-		return nil, fmt.Errorf("error checking iptables version: %v", err)
+		return nil, fmt.Errorf("could not get iptables version: %v", err)
 	}
+	v1, v2, v3, mode, err := extractIptablesVersion(vstring)
+	if err != nil {
+		return nil, fmt.Errorf("failed to extract iptables version from [%s]: %v", vstring, err)
+	}
+
+	checkPresent, waitPresent, randomFullyPresent := getIptablesCommandSupport(v1, v2, v3)
+
 	ipt := IPTables{
-		path:     path,
+		path:           path,
-		proto:    proto,
+		proto:          proto,
-		hasCheck: checkPresent,
+		hasCheck:       checkPresent,
-		hasWait:  waitPresent,
+		hasWait:        waitPresent,
+		hasRandomFully: randomFullyPresent,
+		v1:             v1,
+		v2:             v2,
+		v3:             v3,
+		mode:           mode,
 	}
 	return &ipt, nil
 }
@@ -241,6 +288,63 @@ func (ipt *IPTables) Stats(table, chain string) ([][]string, error) {
 	return rows, nil
 }
 
+// ParseStat parses a single statistic row into a Stat struct. The input should
+// be a string slice that is returned from calling the Stat method.
+func (ipt *IPTables) ParseStat(stat []string) (parsed Stat, err error) {
+	// For forward-compatibility, expect at least 10 fields in the stat
+	if len(stat) < 10 {
+		return parsed, fmt.Errorf("stat contained fewer fields than expected")
+	}
+
+	// Convert the fields that are not plain strings
+	parsed.Packets, err = strconv.ParseUint(stat[0], 0, 64)
+	if err != nil {
+		return parsed, fmt.Errorf(err.Error(), "could not parse packets")
+	}
+	parsed.Bytes, err = strconv.ParseUint(stat[1], 0, 64)
+	if err != nil {
+		return parsed, fmt.Errorf(err.Error(), "could not parse bytes")
+	}
+	_, parsed.Source, err = net.ParseCIDR(stat[7])
+	if err != nil {
+		return parsed, fmt.Errorf(err.Error(), "could not parse source")
+	}
+	_, parsed.Destination, err = net.ParseCIDR(stat[8])
+	if err != nil {
+		return parsed, fmt.Errorf(err.Error(), "could not parse destination")
+	}
+
+	// Put the fields that are strings
+	parsed.Target = stat[2]
+	parsed.Protocol = stat[3]
+	parsed.Opt = stat[4]
+	parsed.Input = stat[5]
+	parsed.Output = stat[6]
+	parsed.Options = stat[9]
+
+	return parsed, nil
+}
+
+// StructuredStats returns statistics as structured data which may be further
+// parsed and marshaled.
+func (ipt *IPTables) StructuredStats(table, chain string) ([]Stat, error) {
+	rawStats, err := ipt.Stats(table, chain)
+	if err != nil {
+		return nil, err
+	}
+
+	structStats := []Stat{}
+	for _, rawStat := range rawStats {
+		stat, err := ipt.ParseStat(rawStat)
+		if err != nil {
+			return nil, err
+		}
+		structStats = append(structStats, stat)
+	}
+
+	return structStats, nil
+}
+
 func (ipt *IPTables) executeList(args []string) ([]string, error) {
 	var stdout bytes.Buffer
 	if err := ipt.runWithOutput(args, &stdout); err != nil {
@@ -248,10 +352,16 @@ func (ipt *IPTables) executeList(args []string) ([]string, error) {
 	}
 
 	rules := strings.Split(stdout.String(), "\n")
+
+	// strip trailing newline
 	if len(rules) > 0 && rules[len(rules)-1] == "" {
 		rules = rules[:len(rules)-1]
 	}
 
+	for i, rule := range rules {
+		rules[i] = filterRuleOutput(rule)
+	}
+
 	return rules, nil
 }
 
@@ -261,6 +371,8 @@ func (ipt *IPTables) NewChain(table, chain string) error {
 	return ipt.run("-t", table, "-N", chain)
 }
 
+const existsErr = 1
+
 // ClearChain flushed (deletes all rules) in the specified table/chain.
 // If the chain does not exist, a new one will be created
 func (ipt *IPTables) ClearChain(table, chain string) error {
@@ -270,7 +382,7 @@ func (ipt *IPTables) ClearChain(table, chain string) error {
 	switch {
 	case err == nil:
 		return nil
-	case eok && eerr.ExitStatus() == 1:
+	case eok && eerr.ExitStatus() == existsErr:
 		// chain already exists. Flush (clear) it.
 		return ipt.run("-t", table, "-F", chain)
 	default:
@@ -289,6 +401,21 @@ func (ipt *IPTables) DeleteChain(table, chain string) error {
 	return ipt.run("-t", table, "-X", chain)
 }
 
+// ChangePolicy changes policy on chain to target
+func (ipt *IPTables) ChangePolicy(table, chain, target string) error {
+	return ipt.run("-t", table, "-P", chain, target)
+}
+
+// Check if the underlying iptables command supports the --random-fully flag
+func (ipt *IPTables) HasRandomFully() bool {
+	return ipt.hasRandomFully
+}
+
+// Return version components of the underlying iptables command
+func (ipt *IPTables) GetIptablesVersion() (int, int, int) {
+	return ipt.v1, ipt.v2, ipt.v3
+}
+
 // run runs an iptables command with the given arguments, ignoring
 // any stdout output
 func (ipt *IPTables) run(args ...string) error {
@@ -308,6 +435,7 @@ func (ipt *IPTables) runWithOutput(args []string, stdout io.Writer) error {
 		}
 		ul, err := fmu.tryLock()
 		if err != nil {
+			syscall.Close(fmu.fd)
 			return err
 		}
 		defer ul.Unlock()
@@ -324,7 +452,7 @@ func (ipt *IPTables) runWithOutput(args []string, stdout io.Writer) error {
 	if err := cmd.Run(); err != nil {
 		switch e := err.(type) {
 		case *exec.ExitError:
-			return &Error{*e, cmd, stderr.String()}
+			return &Error{*e, cmd, stderr.String(), ipt.proto, nil}
 		default:
 			return err
 		}
@@ -343,45 +471,40 @@ func getIptablesCommand(proto Protocol) string {
 }
 
 // Checks if iptables has the "-C" and "--wait" flag
-func getIptablesCommandSupport(path string) (bool, bool, error) {
+func getIptablesCommandSupport(v1 int, v2 int, v3 int) (bool, bool, bool) {
-	vstring, err := getIptablesVersionString(path)
+	return iptablesHasCheckCommand(v1, v2, v3), iptablesHasWaitCommand(v1, v2, v3), iptablesHasRandomFully(v1, v2, v3)
-	if err != nil {
-		return false, false, err
-	}
-
-	v1, v2, v3, err := extractIptablesVersion(vstring)
-	if err != nil {
-		return false, false, err
-	}
-
-	return iptablesHasCheckCommand(v1, v2, v3), iptablesHasWaitCommand(v1, v2, v3), nil
 }
 
-// getIptablesVersion returns the first three components of the iptables version.
+// getIptablesVersion returns the first three components of the iptables version
-// e.g. "iptables v1.3.66" would return (1, 3, 66, nil)
+// and the operating mode (e.g. nf_tables or legacy)
-func extractIptablesVersion(str string) (int, int, int, error) {
+// e.g. "iptables v1.3.66" would return (1, 3, 66, legacy, nil)
-	versionMatcher := regexp.MustCompile("v([0-9]+)\\.([0-9]+)\\.([0-9]+)")
+func extractIptablesVersion(str string) (int, int, int, string, error) {
+	versionMatcher := regexp.MustCompile(`v([0-9]+)\.([0-9]+)\.([0-9]+)(?:\s+\((\w+))?`)
 	result := versionMatcher.FindStringSubmatch(str)
 	if result == nil {
-		return 0, 0, 0, fmt.Errorf("no iptables version found in string: %s", str)
+		return 0, 0, 0, "", fmt.Errorf("no iptables version found in string: %s", str)
 	}
 
 	v1, err := strconv.Atoi(result[1])
 	if err != nil {
-		return 0, 0, 0, err
+		return 0, 0, 0, "", err
 	}
 
 	v2, err := strconv.Atoi(result[2])
 	if err != nil {
-		return 0, 0, 0, err
+		return 0, 0, 0, "", err
 	}
 
 	v3, err := strconv.Atoi(result[3])
 	if err != nil {
-		return 0, 0, 0, err
+		return 0, 0, 0, "", err
 	}
 
-	return v1, v2, v3, nil
+	mode := "legacy"
+	if result[4] != "" {
+		mode = result[4]
+	}
+	return v1, v2, v3, mode, nil
 }
 
 // Runs "iptables --version" to get the version string
@@ -424,6 +547,20 @@ func iptablesHasWaitCommand(v1 int, v2 int, v3 int) bool {
 	return false
 }
 
+// Checks if an iptables version is after 1.6.2, when --random-fully was added
+func iptablesHasRandomFully(v1 int, v2 int, v3 int) bool {
+	if v1 > 1 {
+		return true
+	}
+	if v1 == 1 && v2 > 6 {
+		return true
+	}
+	if v1 == 1 && v2 == 6 && v3 >= 2 {
+		return true
+	}
+	return false
+}
+
 // Checks if a rule specification exists for a table
 func (ipt *IPTables) existsForOldIptables(table, chain string, rulespec []string) (bool, error) {
 	rs := strings.Join(append([]string{"-A", chain}, rulespec...), " ")
@@ -435,3 +572,26 @@ func (ipt *IPTables) existsForOldIptables(table, chain string, rulespec []string
 	}
 	return strings.Contains(stdout.String(), rs), nil
 }
+
+// counterRegex is the regex used to detect nftables counter format
+var counterRegex = regexp.MustCompile(`^\[([0-9]+):([0-9]+)\] `)
+
+// filterRuleOutput works around some inconsistencies in output.
+// For example, when iptables is in legacy vs. nftables mode, it produces
+// different results.
+func filterRuleOutput(rule string) string {
+	out := rule
+
+	// work around an output difference in nftables mode where counters
+	// are output in iptables-save format, rather than iptables -S format
+	// The string begins with "[0:0]"
+	//
+	// Fixes #49
+	if groups := counterRegex.FindStringSubmatch(out); groups != nil {
+		// drop the brackets
+		out = out[len(groups[0]):]
+		out = fmt.Sprintf("%s -c %s %s", out, groups[1], groups[2])
+	}
+
+	return out
+}