diff --git a/Gopkg.lock b/Gopkg.lock index 5156ad21..087428ba 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -98,12 +98,12 @@ version = "v0.5.2" [[projects]] - digest = "1:df4af3a8e15cd72b068b8d898598001935edce4eac4eb5a3d7fd906036123ff4" + digest = "1:2debc00eaf8ffdbfb0570a791a84e64a3cfc146f5e61b54d4abd65c2de364552" name = "github.com/coreos/go-iptables" packages = ["iptables"] pruneopts = "UT" - revision = "259c8e6a4275d497442c721fa52204d7a58bde8b" - version = "v0.2.0" + revision = "f901d6c2a4f2a4df092b98c33366dfba1f93d7a0" + version = "v0.4.5" [[projects]] digest = "1:a2c1d0e43bd3baaa071d1b9ed72c27d78169b2b269f71c105ac4ba34b1be4a39" @@ -1135,6 +1135,7 @@ "k8s.io/api/core/v1", "k8s.io/api/extensions/v1beta1", "k8s.io/api/networking/v1", + "k8s.io/apimachinery/pkg/api/resource", "k8s.io/apimachinery/pkg/apis/meta/v1", "k8s.io/apimachinery/pkg/labels", "k8s.io/apimachinery/pkg/util/intstr", diff --git a/Gopkg.toml b/Gopkg.toml index 99e73e6d..d1f0e188 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -37,7 +37,7 @@ required = ["github.com/osrg/gobgp/gobgp"] [[constraint]] name = "github.com/coreos/go-iptables" - version = "0.2.0" + version = "0.4.1" [[constraint]] branch = "master" diff --git a/pkg/controllers/proxy/network_services_controller.go b/pkg/controllers/proxy/network_services_controller.go index c23bfd41..f31fff5f 100644 --- a/pkg/controllers/proxy/network_services_controller.go +++ b/pkg/controllers/proxy/network_services_controller.go @@ -1225,6 +1225,9 @@ func (nsc *NetworkServicesController) ensureMasqueradeIptablesRule() error { return errors.New("Failed to initialize iptables executor" + err.Error()) } var args = []string{"-m", "ipvs", "--ipvs", "--vdir", "ORIGINAL", "--vmethod", "MASQ", "-m", "comment", "--comment", "", "-j", "SNAT", "--to-source", nsc.nodeIP.String()} + if iptablesCmdHandler.HasRandomFully() { + args = append(args, "--random-fully") + } if nsc.masqueradeAll { err = iptablesCmdHandler.AppendUnique("nat", "POSTROUTING", args...) if err != nil { @@ -1248,6 +1251,10 @@ func (nsc *NetworkServicesController) ensureMasqueradeIptablesRule() error { //TODO: ipset should be used for destination podCidr(s) match after multiple podCidr(s) per node get supported args = []string{"-m", "ipvs", "--ipvs", "--vdir", "ORIGINAL", "--vmethod", "MASQ", "-m", "comment", "--comment", "", "!", "-s", nsc.podCidr, "!", "-d", nsc.podCidr, "-j", "SNAT", "--to-source", nsc.nodeIP.String()} + if iptablesCmdHandler.HasRandomFully() { + args = append(args, "--random-fully") + } + err = iptablesCmdHandler.AppendUnique("nat", "POSTROUTING", args...) if err != nil { return errors.New("Failed to run iptables command" + err.Error()) @@ -1269,6 +1276,16 @@ func (nsc *NetworkServicesController) deleteBadMasqueradeIptablesRules() error { {"-m", "ipvs", "--ipvs", "--vdir", "ORIGINAL", "--vmethod", "MASQ", "-m", "comment", "--comment", "", "!", "-s", nsc.podCidr, "!", "-d", nsc.podCidr, "-j", "MASQUERADE"}, } + // If random fully is supported remove the original rules as well + if iptablesCmdHandler.HasRandomFully() { + argsBad = append(argsBad, []string{"-m", "ipvs", "--ipvs", "--vdir", "ORIGINAL", "--vmethod", "MASQ", "-m", "comment", "--comment", "", "-j", "SNAT", "--to-source", nsc.nodeIP.String()}) + + if len(nsc.podCidr) > 0 { + argsBad = append(argsBad, []string{"-m", "ipvs", "--ipvs", "--vdir", "ORIGINAL", "--vmethod", "MASQ", "-m", "comment", "--comment", "", + "!", "-s", nsc.podCidr, "!", "-d", nsc.podCidr, "-j", "SNAT", "--to-source", nsc.nodeIP.String()}) + } + } + for _, args := range argsBad { exists, err := iptablesCmdHandler.Exists("nat", "POSTROUTING", args...) if err != nil { diff --git a/pkg/controllers/routing/pod_egress.go b/pkg/controllers/routing/pod_egress.go index 7122ed63..1fa1e261 100644 --- a/pkg/controllers/routing/pod_egress.go +++ b/pkg/controllers/routing/pod_egress.go @@ -36,6 +36,10 @@ func (nrc *NetworkRoutingController) createPodEgressRule() error { if nrc.isIpv6 { podEgressArgs = podEgressArgs6 } + if iptablesCmdHandler.HasRandomFully() { + podEgressArgs = append(podEgressArgs, "--random-fully") + } + err = iptablesCmdHandler.AppendUnique("nat", "POSTROUTING", podEgressArgs...) if err != nil { return errors.New("Failed to add iptables rule to masquerade outbound traffic from pods: " + @@ -57,6 +61,10 @@ func (nrc *NetworkRoutingController) deletePodEgressRule() error { if nrc.isIpv6 { podEgressArgs = podEgressArgs6 } + if iptablesCmdHandler.HasRandomFully() { + podEgressArgs = append(podEgressArgs, "--random-fully") + } + exists, err := iptablesCmdHandler.Exists("nat", "POSTROUTING", podEgressArgs...) if err != nil { return errors.New("Failed to lookup iptables rule to masquerade outbound traffic from pods: " + err.Error()) @@ -83,6 +91,16 @@ func (nrc *NetworkRoutingController) deleteBadPodEgressRules() error { if nrc.isIpv6 { podEgressArgsBad = podEgressArgsBad6 } + + // If random fully is supported remove the original rule as well + if iptablesCmdHandler.HasRandomFully() { + if !nrc.isIpv6 { + podEgressArgsBad = append(podEgressArgsBad, podEgressArgs4) + } else { + podEgressArgsBad = append(podEgressArgsBad, podEgressArgs6) + } + } + for _, args := range podEgressArgsBad { exists, err := iptablesCmdHandler.Exists("nat", "POSTROUTING", args...) if err != nil { diff --git a/vendor/github.com/coreos/go-iptables/NOTICE b/vendor/github.com/coreos/go-iptables/NOTICE new file mode 100644 index 00000000..23a0ada2 --- /dev/null +++ b/vendor/github.com/coreos/go-iptables/NOTICE @@ -0,0 +1,5 @@ +CoreOS Project +Copyright 2018 CoreOS, Inc + +This product includes software developed at CoreOS, Inc. +(http://www.coreos.com/). diff --git a/vendor/github.com/coreos/go-iptables/iptables/iptables.go b/vendor/github.com/coreos/go-iptables/iptables/iptables.go index 1d8b78e2..1074275b 100644 --- a/vendor/github.com/coreos/go-iptables/iptables/iptables.go +++ b/vendor/github.com/coreos/go-iptables/iptables/iptables.go @@ -29,11 +29,16 @@ import ( // Adds the output of stderr to exec.ExitError type Error struct { exec.ExitError - cmd exec.Cmd - msg string + cmd exec.Cmd + msg string + proto Protocol + exitStatus *int //for overriding } func (e *Error) ExitStatus() int { + if e.exitStatus != nil { + return *e.exitStatus + } return e.Sys().(syscall.WaitStatus).ExitStatus() } @@ -41,6 +46,17 @@ func (e *Error) Error() string { return fmt.Sprintf("running %v: exit status %v: %v", e.cmd.Args, e.ExitStatus(), e.msg) } +// IsNotExist returns true if the error is due to the chain or rule not existing +func (e *Error) IsNotExist() bool { + if e.ExitStatus() != 1 { + return false + } + cmdIptables := getIptablesCommand(e.proto) + msgNoRuleExist := fmt.Sprintf("%s: Bad rule (does a matching rule exist in that chain?).\n", cmdIptables) + msgNoChainExist := fmt.Sprintf("%s: No chain/target/match by that name.\n", cmdIptables) + return strings.Contains(e.msg, msgNoRuleExist) || strings.Contains(e.msg, msgNoChainExist) +} + // Protocol to differentiate between IPv4 and IPv6 type Protocol byte @@ -50,10 +66,29 @@ const ( ) type IPTables struct { - path string - proto Protocol - hasCheck bool - hasWait bool + path string + proto Protocol + hasCheck bool + hasWait bool + hasRandomFully bool + v1 int + v2 int + v3 int + mode string // the underlying iptables operating mode, e.g. nf_tables +} + +// Stat represents a structured statistic entry. +type Stat struct { + Packets uint64 `json:"pkts"` + Bytes uint64 `json:"bytes"` + Target string `json:"target"` + Protocol string `json:"prot"` + Opt string `json:"opt"` + Input string `json:"in"` + Output string `json:"out"` + Source *net.IPNet `json:"source"` + Destination *net.IPNet `json:"destination"` + Options string `json:"options"` } // New creates a new IPTables. @@ -69,15 +104,27 @@ func NewWithProtocol(proto Protocol) (*IPTables, error) { if err != nil { return nil, err } - checkPresent, waitPresent, err := getIptablesCommandSupport(path) + vstring, err := getIptablesVersionString(path) if err != nil { - return nil, fmt.Errorf("error checking iptables version: %v", err) + return nil, fmt.Errorf("could not get iptables version: %v", err) } + v1, v2, v3, mode, err := extractIptablesVersion(vstring) + if err != nil { + return nil, fmt.Errorf("failed to extract iptables version from [%s]: %v", vstring, err) + } + + checkPresent, waitPresent, randomFullyPresent := getIptablesCommandSupport(v1, v2, v3) + ipt := IPTables{ - path: path, - proto: proto, - hasCheck: checkPresent, - hasWait: waitPresent, + path: path, + proto: proto, + hasCheck: checkPresent, + hasWait: waitPresent, + hasRandomFully: randomFullyPresent, + v1: v1, + v2: v2, + v3: v3, + mode: mode, } return &ipt, nil } @@ -241,6 +288,63 @@ func (ipt *IPTables) Stats(table, chain string) ([][]string, error) { return rows, nil } +// ParseStat parses a single statistic row into a Stat struct. The input should +// be a string slice that is returned from calling the Stat method. +func (ipt *IPTables) ParseStat(stat []string) (parsed Stat, err error) { + // For forward-compatibility, expect at least 10 fields in the stat + if len(stat) < 10 { + return parsed, fmt.Errorf("stat contained fewer fields than expected") + } + + // Convert the fields that are not plain strings + parsed.Packets, err = strconv.ParseUint(stat[0], 0, 64) + if err != nil { + return parsed, fmt.Errorf(err.Error(), "could not parse packets") + } + parsed.Bytes, err = strconv.ParseUint(stat[1], 0, 64) + if err != nil { + return parsed, fmt.Errorf(err.Error(), "could not parse bytes") + } + _, parsed.Source, err = net.ParseCIDR(stat[7]) + if err != nil { + return parsed, fmt.Errorf(err.Error(), "could not parse source") + } + _, parsed.Destination, err = net.ParseCIDR(stat[8]) + if err != nil { + return parsed, fmt.Errorf(err.Error(), "could not parse destination") + } + + // Put the fields that are strings + parsed.Target = stat[2] + parsed.Protocol = stat[3] + parsed.Opt = stat[4] + parsed.Input = stat[5] + parsed.Output = stat[6] + parsed.Options = stat[9] + + return parsed, nil +} + +// StructuredStats returns statistics as structured data which may be further +// parsed and marshaled. +func (ipt *IPTables) StructuredStats(table, chain string) ([]Stat, error) { + rawStats, err := ipt.Stats(table, chain) + if err != nil { + return nil, err + } + + structStats := []Stat{} + for _, rawStat := range rawStats { + stat, err := ipt.ParseStat(rawStat) + if err != nil { + return nil, err + } + structStats = append(structStats, stat) + } + + return structStats, nil +} + func (ipt *IPTables) executeList(args []string) ([]string, error) { var stdout bytes.Buffer if err := ipt.runWithOutput(args, &stdout); err != nil { @@ -248,10 +352,16 @@ func (ipt *IPTables) executeList(args []string) ([]string, error) { } rules := strings.Split(stdout.String(), "\n") + + // strip trailing newline if len(rules) > 0 && rules[len(rules)-1] == "" { rules = rules[:len(rules)-1] } + for i, rule := range rules { + rules[i] = filterRuleOutput(rule) + } + return rules, nil } @@ -261,6 +371,8 @@ func (ipt *IPTables) NewChain(table, chain string) error { return ipt.run("-t", table, "-N", chain) } +const existsErr = 1 + // ClearChain flushed (deletes all rules) in the specified table/chain. // If the chain does not exist, a new one will be created func (ipt *IPTables) ClearChain(table, chain string) error { @@ -270,7 +382,7 @@ func (ipt *IPTables) ClearChain(table, chain string) error { switch { case err == nil: return nil - case eok && eerr.ExitStatus() == 1: + case eok && eerr.ExitStatus() == existsErr: // chain already exists. Flush (clear) it. return ipt.run("-t", table, "-F", chain) default: @@ -289,6 +401,21 @@ func (ipt *IPTables) DeleteChain(table, chain string) error { return ipt.run("-t", table, "-X", chain) } +// ChangePolicy changes policy on chain to target +func (ipt *IPTables) ChangePolicy(table, chain, target string) error { + return ipt.run("-t", table, "-P", chain, target) +} + +// Check if the underlying iptables command supports the --random-fully flag +func (ipt *IPTables) HasRandomFully() bool { + return ipt.hasRandomFully +} + +// Return version components of the underlying iptables command +func (ipt *IPTables) GetIptablesVersion() (int, int, int) { + return ipt.v1, ipt.v2, ipt.v3 +} + // run runs an iptables command with the given arguments, ignoring // any stdout output func (ipt *IPTables) run(args ...string) error { @@ -308,6 +435,7 @@ func (ipt *IPTables) runWithOutput(args []string, stdout io.Writer) error { } ul, err := fmu.tryLock() if err != nil { + syscall.Close(fmu.fd) return err } defer ul.Unlock() @@ -324,7 +452,7 @@ func (ipt *IPTables) runWithOutput(args []string, stdout io.Writer) error { if err := cmd.Run(); err != nil { switch e := err.(type) { case *exec.ExitError: - return &Error{*e, cmd, stderr.String()} + return &Error{*e, cmd, stderr.String(), ipt.proto, nil} default: return err } @@ -343,45 +471,40 @@ func getIptablesCommand(proto Protocol) string { } // Checks if iptables has the "-C" and "--wait" flag -func getIptablesCommandSupport(path string) (bool, bool, error) { - vstring, err := getIptablesVersionString(path) - if err != nil { - return false, false, err - } - - v1, v2, v3, err := extractIptablesVersion(vstring) - if err != nil { - return false, false, err - } - - return iptablesHasCheckCommand(v1, v2, v3), iptablesHasWaitCommand(v1, v2, v3), nil +func getIptablesCommandSupport(v1 int, v2 int, v3 int) (bool, bool, bool) { + return iptablesHasCheckCommand(v1, v2, v3), iptablesHasWaitCommand(v1, v2, v3), iptablesHasRandomFully(v1, v2, v3) } -// getIptablesVersion returns the first three components of the iptables version. -// e.g. "iptables v1.3.66" would return (1, 3, 66, nil) -func extractIptablesVersion(str string) (int, int, int, error) { - versionMatcher := regexp.MustCompile("v([0-9]+)\\.([0-9]+)\\.([0-9]+)") +// getIptablesVersion returns the first three components of the iptables version +// and the operating mode (e.g. nf_tables or legacy) +// e.g. "iptables v1.3.66" would return (1, 3, 66, legacy, nil) +func extractIptablesVersion(str string) (int, int, int, string, error) { + versionMatcher := regexp.MustCompile(`v([0-9]+)\.([0-9]+)\.([0-9]+)(?:\s+\((\w+))?`) result := versionMatcher.FindStringSubmatch(str) if result == nil { - return 0, 0, 0, fmt.Errorf("no iptables version found in string: %s", str) + return 0, 0, 0, "", fmt.Errorf("no iptables version found in string: %s", str) } v1, err := strconv.Atoi(result[1]) if err != nil { - return 0, 0, 0, err + return 0, 0, 0, "", err } v2, err := strconv.Atoi(result[2]) if err != nil { - return 0, 0, 0, err + return 0, 0, 0, "", err } v3, err := strconv.Atoi(result[3]) if err != nil { - return 0, 0, 0, err + return 0, 0, 0, "", err } - return v1, v2, v3, nil + mode := "legacy" + if result[4] != "" { + mode = result[4] + } + return v1, v2, v3, mode, nil } // Runs "iptables --version" to get the version string @@ -424,6 +547,20 @@ func iptablesHasWaitCommand(v1 int, v2 int, v3 int) bool { return false } +// Checks if an iptables version is after 1.6.2, when --random-fully was added +func iptablesHasRandomFully(v1 int, v2 int, v3 int) bool { + if v1 > 1 { + return true + } + if v1 == 1 && v2 > 6 { + return true + } + if v1 == 1 && v2 == 6 && v3 >= 2 { + return true + } + return false +} + // Checks if a rule specification exists for a table func (ipt *IPTables) existsForOldIptables(table, chain string, rulespec []string) (bool, error) { rs := strings.Join(append([]string{"-A", chain}, rulespec...), " ") @@ -435,3 +572,26 @@ func (ipt *IPTables) existsForOldIptables(table, chain string, rulespec []string } return strings.Contains(stdout.String(), rs), nil } + +// counterRegex is the regex used to detect nftables counter format +var counterRegex = regexp.MustCompile(`^\[([0-9]+):([0-9]+)\] `) + +// filterRuleOutput works around some inconsistencies in output. +// For example, when iptables is in legacy vs. nftables mode, it produces +// different results. +func filterRuleOutput(rule string) string { + out := rule + + // work around an output difference in nftables mode where counters + // are output in iptables-save format, rather than iptables -S format + // The string begins with "[0:0]" + // + // Fixes #49 + if groups := counterRegex.FindStringSubmatch(out); groups != nil { + // drop the brackets + out = out[len(groups[0]):] + out = fmt.Sprintf("%s -c %s %s", out, groups[1], groups[2]) + } + + return out +}