Merge pull request #169 from ryarnyah/fix/clean-ipset

Delete ipset dependency + delete unused ipsets

app/controllers/network_policy_controller.go

@@ -18,7 +18,6 @@ import (
 	"github.com/cloudnativelabs/kube-router/utils"
 	"github.com/coreos/go-iptables/iptables"
 	"github.com/golang/glog"
-	"github.com/janeczku/go-ipset/ipset"
 	"k8s.io/client-go/kubernetes"
 	api "k8s.io/client-go/pkg/api/v1"
 	apiv1 "k8s.io/client-go/pkg/api/v1"
@@ -46,6 +45,7 @@ type NetworkPolicyController struct {
 
 	// list of all active network policies expressed as networkPolicyInfo
 	networkPoliciesInfo *[]networkPolicyInfo
+	ipset               *utils.IPSet
 }
 
 // internal structure to represent a network policy
@@ -192,7 +192,7 @@ func (npc *NetworkPolicyController) Sync() error {
 
 	}
 
-	activePolicyChains, err := npc.syncNetworkPolicyChains()
+	activePolicyChains, activePolicyIpSets, err := npc.syncNetworkPolicyChains()
 	if err != nil {
 		return errors.New("Aborting sync. Failed to sync network policy chains: " + err.Error())
 	}
@@ -202,7 +202,7 @@ func (npc *NetworkPolicyController) Sync() error {
 		return errors.New("Aborting sync. Failed to sync pod firewalls: " + err.Error())
 	}
 
-	err = cleanupStaleRules(activePolicyChains, activePodFwChains)
+	err = cleanupStaleRules(activePolicyChains, activePodFwChains, activePolicyIpSets)
 	if err != nil {
 		return errors.New("Aborting sync. Failed to cleanup stale iptable rules: " + err.Error())
 	}
@@ -215,9 +215,10 @@ func (npc *NetworkPolicyController) Sync() error {
 // is used for matching destination ip address. Each ingress rule in the network
 // policyspec is evaluated to set of matching pods, which are grouped in to a
 // ipset used for source ip addr matching.
-func (npc *NetworkPolicyController) syncNetworkPolicyChains() (map[string]bool, error) {
+func (npc *NetworkPolicyController) syncNetworkPolicyChains() (map[string]bool, map[string]bool, error) {
 
 	activePolicyChains := make(map[string]bool)
+	activePolicyIpSets := make(map[string]bool)
 
 	iptablesCmdHandler, err := iptables.New()
 	if err != nil {
@@ -231,31 +232,34 @@ func (npc *NetworkPolicyController) syncNetworkPolicyChains() (map[string]bool,
 		policyChainName := networkPolicyChainName(policy.namespace, policy.name)
 		err := iptablesCmdHandler.NewChain("filter", policyChainName)
 		if err != nil && err.(*iptables.Error).ExitStatus() != 1 {
-			return nil, fmt.Errorf("Failed to run iptables command: %s", err.Error())
+			return nil, nil, fmt.Errorf("Failed to run iptables command: %s", err.Error())
 		}
 
 		activePolicyChains[policyChainName] = true
 
 		// create a ipset for all destination pod ip's matched by the policy spec PodSelector
 		destPodIpSetName := policyDestinationPodIpSetName(policy.namespace, policy.name)
-		destPodIpSet, err := ipset.New(destPodIpSetName, "hash:ip", &ipset.Params{})
+		destPodIpSet, err := npc.ipset.Create(destPodIpSetName, utils.TypeHashIP, utils.OptionTimeout, "0")
 		if err != nil {
-			return nil, fmt.Errorf("failed to create ipset: %s", err.Error())
+			return nil, nil, fmt.Errorf("failed to create ipset: %s", err.Error())
 		}
 
 		// flush all entries in the set
 		if destPodIpSet.Flush() != nil {
-			return nil, fmt.Errorf("failed to flush ipset while syncing iptables: %s", err.Error())
+			return nil, nil, fmt.Errorf("failed to flush ipset while syncing iptables: %s", err.Error())
 		}
+
+		activePolicyIpSets[destPodIpSet.Name] = true
+
 		for k := range policy.destPods {
 			// TODO restrict ipset to ip's of pods running on the node
-			destPodIpSet.Add(k, 0)
+			destPodIpSet.Add(k, utils.OptionTimeout, "0")
 		}
 
 		// TODO use iptables-restore to better implement the logic, than flush and add rules
 		err = iptablesCmdHandler.ClearChain("filter", policyChainName)
 		if err != nil && err.(*iptables.Error).ExitStatus() != 1 {
-			return nil, fmt.Errorf("Failed to run iptables command: %s", err.Error())
+			return nil, nil, fmt.Errorf("Failed to run iptables command: %s", err.Error())
 		}
 
 		// From network policy spec: "If field 'Ingress' is empty then this NetworkPolicy does not allow any traffic "
@@ -270,17 +274,19 @@ func (npc *NetworkPolicyController) syncNetworkPolicyChains() (map[string]bool,
 
 			if len(ingressRule.srcPods) != 0 {
 				srcPodIpSetName := policySourcePodIpSetName(policy.namespace, policy.name, i)
-				srcPodIpSet, err := ipset.New(srcPodIpSetName, "hash:ip", &ipset.Params{})
+				srcPodIpSet, err := npc.ipset.Create(srcPodIpSetName, utils.TypeHashIP, utils.OptionTimeout, "0")
 				if err != nil {
-					return nil, fmt.Errorf("failed to create ipset: %s", err.Error())
+					return nil, nil, fmt.Errorf("failed to create ipset: %s", err.Error())
 				}
 				// flush all entries in the set
 				if srcPodIpSet.Flush() != nil {
-					return nil, fmt.Errorf("failed to flush ipset while syncing iptables: %s", err.Error())
+					return nil, nil, fmt.Errorf("failed to flush ipset while syncing iptables: %s", err.Error())
 				}
 
+				activePolicyIpSets[srcPodIpSet.Name] = true
+
 				for _, pod := range ingressRule.srcPods {
-					srcPodIpSet.Add(pod.ip, 0)
+					srcPodIpSet.Add(pod.ip, utils.OptionTimeout, "0")
 				}
 
 				if len(ingressRule.ports) != 0 {
@@ -297,7 +303,7 @@ func (npc *NetworkPolicyController) syncNetworkPolicyChains() (map[string]bool,
 							"-j", "ACCEPT"}
 						err := iptablesCmdHandler.AppendUnique("filter", policyChainName, args...)
 						if err != nil {
-							return nil, fmt.Errorf("Failed to run iptables command: %s", err.Error())
+							return nil, nil, fmt.Errorf("Failed to run iptables command: %s", err.Error())
 						}
 					}
 				} else {
@@ -311,7 +317,7 @@ func (npc *NetworkPolicyController) syncNetworkPolicyChains() (map[string]bool,
 						"-j", "ACCEPT"}
 					err := iptablesCmdHandler.AppendUnique("filter", policyChainName, args...)
 					if err != nil {
-						return nil, fmt.Errorf("Failed to run iptables command: %s", err.Error())
+						return nil, nil, fmt.Errorf("Failed to run iptables command: %s", err.Error())
 					}
 				}
 			}
@@ -329,7 +335,7 @@ func (npc *NetworkPolicyController) syncNetworkPolicyChains() (map[string]bool,
 						"-j", "ACCEPT"}
 					err := iptablesCmdHandler.AppendUnique("filter", policyChainName, args...)
 					if err != nil {
-						return nil, fmt.Errorf("Failed to run iptables command: %s", err.Error())
+						return nil, nil, fmt.Errorf("Failed to run iptables command: %s", err.Error())
 					}
 				}
 			}
@@ -344,7 +350,7 @@ func (npc *NetworkPolicyController) syncNetworkPolicyChains() (map[string]bool,
 					"-j", "ACCEPT"}
 				err := iptablesCmdHandler.AppendUnique("filter", policyChainName, args...)
 				if err != nil {
-					return nil, fmt.Errorf("Failed to run iptables command: %s", err.Error())
+					return nil, nil, fmt.Errorf("Failed to run iptables command: %s", err.Error())
 				}
 			}
 		}
@@ -352,7 +358,7 @@ func (npc *NetworkPolicyController) syncNetworkPolicyChains() (map[string]bool,
 
 	glog.Infof("Iptables chains in the filter table are synchronized with the network policies.")
 
-	return activePolicyChains, nil
+	return activePolicyChains, activePolicyIpSets, nil
 }
 
 func (npc *NetworkPolicyController) syncPodFirewallChains() (map[string]bool, error) {
@@ -478,15 +484,24 @@ func (npc *NetworkPolicyController) syncPodFirewallChains() (map[string]bool, er
 	return activePodFwChains, nil
 }
 
-func cleanupStaleRules(activePolicyChains, activePodFwChains map[string]bool) error {
+func cleanupStaleRules(activePolicyChains, activePodFwChains, activePolicyIPSets map[string]bool) error {
 
 	cleanupPodFwChains := make([]string, 0)
 	cleanupPolicyChains := make([]string, 0)
+	cleanupPolicyIPSets := make([]*utils.Set, 0)
 
 	iptablesCmdHandler, err := iptables.New()
 	if err != nil {
 		glog.Fatalf("failed to initialize iptables command executor due to %s", err.Error())
 	}
+	ipsets, err := utils.NewIPSet()
+	if err != nil {
+		glog.Fatalf("failed to create ipsets command executor due to %s", err.Error())
+	}
+	err = ipsets.Save()
+	if err != nil {
+		glog.Fatalf("failed to initialize ipsets command executor due to %s", err.Error())
+	}
 
 	// get the list of chains created for pod firewall and network policies
 	chains, err := iptablesCmdHandler.ListChains("filter")
@@ -502,6 +517,14 @@ func cleanupStaleRules(activePolicyChains, activePodFwChains map[string]bool) er
 			}
 		}
 	}
+	for _, set := range ipsets.Sets {
+		if strings.HasPrefix(set.Name, "KUBE-SRC-") ||
+			strings.HasPrefix(set.Name, "KUBE-DST-") {
+			if _, ok := activePolicyIPSets[set.Name]; !ok {
+				cleanupPolicyIPSets = append(cleanupPolicyIPSets, set)
+			}
+		}
+	}
 
 	// cleanup FORWARD chain rules to jump to pod firewall
 	for _, chain := range cleanupPodFwChains {
@@ -584,7 +607,13 @@ func cleanupStaleRules(activePolicyChains, activePodFwChains map[string]bool) er
 		glog.Infof("Deleted network policy chain: %s from the filter table", policyChain)
 	}
 
-	// TODO delete unused ipsets
+	// cleanup network policy ipsets
+	for _, set := range cleanupPolicyIPSets {
+		err = set.Destroy()
+		if err != nil {
+			return fmt.Errorf("Failed to delete ipset %s due to %s", set.Name, err)
+		}
+	}
 	return nil
 }
 
@@ -932,7 +961,7 @@ func (npc *NetworkPolicyController) Cleanup() {
 	}
 
 	// delete all ipsets
-	err = ipset.DestroyAll()
+	err = npc.ipset.Destroy()
 	if err != nil {
 		glog.Errorf("Failed to clean up ipsets: " + err.Error())
 	}
@@ -966,6 +995,16 @@ func NewNetworkPolicyController(clientset *kubernetes.Clientset, config *options
 	}
 	npc.nodeIP = nodeIP
 
+	ipset, err := utils.NewIPSet()
+	if err != nil {
+		return nil, err
+	}
+	err = ipset.Save()
+	if err != nil {
+		return nil, err
+	}
+	npc.ipset = ipset
+
 	watchers.PodWatcher.RegisterHandler(&npc)
 	watchers.NetworkPolicyWatcher.RegisterHandler(&npc)
 	watchers.NamespaceWatcher.RegisterHandler(&npc)

app/controllers/network_routes_controller.go

@@ -23,7 +23,6 @@ import (
 	"github.com/cloudnativelabs/kube-router/utils"
 	"github.com/coreos/go-iptables/iptables"
 	"github.com/golang/glog"
-	"github.com/janeczku/go-ipset/ipset"
 	bgpapi "github.com/osrg/gobgp/api"
 	"github.com/osrg/gobgp/config"
 	"github.com/osrg/gobgp/packet/bgp"
@@ -53,7 +52,7 @@ type NetworkRoutingController struct {
 	globalPeerRouters    []*config.NeighborConfig
 	nodePeerRouters      []string
 	bgpFullMeshMode      bool
-	podSubnetsIpSet      *ipset.IPSet
+	podSubnetsIpSet      *utils.Set
 	enableOverlays       bool
 }
 
@@ -673,8 +672,14 @@ func deletePodSubnetIpSet() error {
 		return errors.New("Ensure ipset package is installed: " + err.Error())
 	}
 
-	podSubnetIpSet := ipset.IPSet{Name: podSubnetIpSetName, HashType: "bitmap:ip"}
+	ipset, err := utils.NewIPSet()
-	err = podSubnetIpSet.Destroy()
+	if err != nil {
+		return err
+	}
+	ipset.Sets = append(ipset.Sets, &utils.Set{
+		Name: podSubnetIpSetName,
+	})
+	err = ipset.Destroy()
 	if err != nil {
 		return errors.New("Failure deleting Pod egress ipset: " + err.Error())
 	}
@@ -738,7 +743,7 @@ func (nrc *NetworkRoutingController) syncPodSubnetIpSet() error {
 		currentPodCidrs = append(currentPodCidrs, node.Spec.PodCIDR)
 	}
 
-	err = nrc.podSubnetsIpSet.Refresh(currentPodCidrs)
+	err = nrc.podSubnetsIpSet.Refresh(currentPodCidrs, utils.OptionTimeout, "0")
 	if err != nil {
 		return errors.New("Failed to update Pod subnet ipset: " + err.Error())
 	}
@@ -1143,19 +1148,19 @@ func NewNetworkRoutingController(clientset *kubernetes.Clientset,
 	nrc.enablePodEgress = kubeRouterConfig.EnablePodEgress
 	nrc.syncPeriod = kubeRouterConfig.RoutesSyncPeriod
 	nrc.clientset = clientset
+	ipset, err := utils.NewIPSet()
+	if err != nil {
+		return nil, err
+	}
 
 	if nrc.enablePodEgress || len(nrc.clusterCIDR) != 0 {
 		nrc.enablePodEgress = true
 
 		// TODO: Add bitmap hashtype support to ipset package. It would work well here.
-		podSubnetIpSet, err := ipset.New(podSubnetIpSetName, "hash:net", &ipset.Params{})
+		nrc.podSubnetsIpSet, err = ipset.Create(podSubnetIpSetName, utils.TypeHashNet, utils.OptionTimeout, "0")
 		if err != nil {
 			return nil, fmt.Errorf("failed to create Pod subnet ipset: %s", err.Error())
 		}
-
-		nrc.podSubnetsIpSet = podSubnetIpSet
-	} else {
-		nrc.podSubnetsIpSet = nil
 	}
 
 	if kubeRouterConfig.ClusterAsn != 0 {

glide.lock

@@ -140,12 +140,6 @@ imports:
   - client/v2
   - models
   - pkg/escape
-- name: github.com/janeczku/go-ipset
-  version: 26fff2141bbf686b96609d5121146c449da5404a
-  repo: git@github.com:/bzub/go-ipset
-  vcs: git
-  subpackages:
-  - ipset
 - name: github.com/jmespath/go-jmespath
   version: bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d
 - name: github.com/juju/ratelimit

glide.yaml

@@ -6,12 +6,6 @@ import:
   - iptables
 - package: github.com/golang/glog
   version: master
-- package: github.com/janeczku/go-ipset
-  version: master
-  repo: git@github.com:/bzub/go-ipset
-  vcs: git
-  subpackages:
-  - ipset
 - package: github.com/docker/libnetwork
   repo: git@github.com:/cloudnativelabs/libnetwork
   vcs: git

utils/ipset.go

@@ -0,0 +1,374 @@
+package utils
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"os/exec"
+	"strings"
+)
+
+var (
+	// Error returned when ipset binary is not found.
+	errIpsetNotFound = errors.New("Ipset utility not found")
+)
+
+const (
+	// FamillyInet IPV4.
+	FamillyInet = "inet"
+	// FamillyInet6 IPV6.
+	FamillyInet6 = "inet6"
+
+	// DefaultMaxElem Default OptionMaxElem value.
+	DefaultMaxElem = "65536"
+	// DefaultHasSize Defaul OptionHashSize value.
+	DefaultHasSize = "1024"
+
+	// TypeHashIP The hash:ip set type uses a hash to store IP host addresses (default) or network addresses. Zero valued IP address cannot be stored in a hash:ip type of set.
+	TypeHashIP = "hash:ip"
+	// TypeHashMac The hash:mac set type uses a hash to store MAC addresses. Zero valued MAC addresses cannot be stored in a hash:mac type of set.
+	TypeHashMac = "hash:mac"
+	// TypeHashNet The hash:net set type uses a hash to store different sized IP network addresses. Network address with zero prefix size cannot be stored in this type of sets.
+	TypeHashNet = "hash:net"
+	// TypeHashNetNet The hash:net,net set type uses a hash to store pairs of different sized IP network addresses. Bear in mind that the first parameter has precedence over the second, so a nomatch entry could be potentially be ineffective if a more specific first parameter existed with a suitable second parameter. Network address with zero prefix size cannot be stored in this type of set.
+	TypeHashNetNet = "hash:net,net"
+	// TypeHashIPPort The hash:ip,port set type uses a hash to store IP address and port number pairs. The port number is interpreted together with a protocol (default TCP) and zero protocol number cannot be used.
+	TypeHashIPPort = "hash:ip,port"
+	// TypeHashNetPort The hash:net,port set type uses a hash to store different sized IP network address and port pairs. The port number is interpreted together with a protocol (default TCP) and zero protocol number cannot be used. Network address with zero prefix size is not accepted either.
+	TypeHashNetPort = "hash:net,port"
+	// TypeHashIPPortIP The hash:ip,port,ip set type uses a hash to store IP address, port number and a second IP address triples. The port number is interpreted together with a protocol (default TCP) and zero protocol number cannot be used.
+	TypeHashIPPortIP = "hash:ip,port,ip"
+	// TypeHashIPPortNet The hash:ip,port,net set type uses a hash to store IP address, port number and IP network address triples. The port number is interpreted together with a protocol (default TCP) and zero protocol number cannot be used. Network address with zero prefix size cannot be stored either.
+	TypeHashIPPortNet = "hash:ip,port,net"
+	// TypeHashIPMark The hash:ip,mark set type uses a hash to store IP address and packet mark pairs.
+	TypeHashIPMark = "hash:ip,mark"
+	// TypeHashIPNetPortNet The hash:net,port,net set type behaves similarly to hash:ip,port,net but accepts a cidr value for both the first and last parameter. Either subnet is permitted to be a /0 should you wish to match port between all destinations.
+	TypeHashIPNetPortNet = "hash:net,port,net"
+	// TypeHashNetIface The hash:net,iface set type uses a hash to store different sized IP network address and interface name pairs.
+	TypeHashNetIface = "hash:net,iface"
+	// TypeListSet The list:set type uses a simple list in which you can store set names.
+	TypeListSet = "list:set"
+
+	// OptionTimeout All set types supports the optional timeout parameter when creating a set and adding entries. The value of the timeout parameter for the create command means the default timeout value (in seconds) for new entries. If a set is created with timeout support, then the same timeout option can be used to specify non-default timeout values when adding entries. Zero timeout value means the entry is added permanent to the set. The timeout value of already added elements can be changed by readding the element using the -exist option. When listing the set, the number of entries printed in the header might be larger than the listed number of entries for sets with the timeout extensions: the number of entries in the set is updated when elements added/deleted to the set and periodically when the garbage colletor evicts the timed out entries.`
+	OptionTimeout = "timeout"
+	// OptionCounters All set types support the optional counters option when creating a set. If the option is specified then the set is created with packet and byte counters per element support. The packet and byte counters are initialized to zero when the elements are (re-)added to the set, unless the packet and byte counter values are explicitly specified by the packets and bytes options. An example when an element is added to a set with non-zero counter values.
+	OptionCounters = "counters"
+	// OptionPackets All set types support the optional counters option when creating a set. If the option is specified then the set is created with packet and byte counters per element support. The packet and byte counters are initialized to zero when the elements are (re-)added to the set, unless the packet and byte counter values are explicitly specified by the packets and bytes options. An example when an element is added to a set with non-zero counter values.
+	OptionPackets = "packets"
+	// OptionBytes All set types support the optional counters option when creating a set. If the option is specified then the set is created with packet and byte counters per element support. The packet and byte counters are initialized to zero when the elements are (re-)added to the set, unless the packet and byte counter values are explicitly specified by the packets and bytes options. An example when an element is added to a set with non-zero counter values.
+	OptionBytes = "bytes"
+	// OptionComment All set types support the optional comment extension. Enabling this extension on an ipset enables you to annotate an ipset entry with an arbitrary string. This string is completely ignored by both the kernel and ipset itself and is purely for providing a convenient means to document the reason for an entry's existence. Comments must not contain any quotation marks and the usual escape character (\) has no meaning
+	OptionComment = "comment"
+	// OptionSkbinfo All set types support the optional skbinfo extension. This extension allow to store the metainfo (firewall mark, tc class and hardware queue) with every entry and map it to packets by usage of SET netfilter target with --map-set option. skbmark option format: MARK or MARK/MASK, where MARK and MASK are 32bit hex numbers with 0x prefix. If only mark is specified mask 0xffffffff are used. skbprio option has tc class format: MAJOR:MINOR, where major and minor numbers are hex without 0x prefix. skbqueue option is just decimal number.
+	OptionSkbinfo = "skbinfo"
+	// OptionSkbmark All set types support the optional skbinfo extension. This extension allow to store the metainfo (firewall mark, tc class and hardware queue) with every entry and map it to packets by usage of SET netfilter target with --map-set option. skbmark option format: MARK or MARK/MASK, where MARK and MASK are 32bit hex numbers with 0x prefix. If only mark is specified mask 0xffffffff are used. skbprio option has tc class format: MAJOR:MINOR, where major and minor numbers are hex without 0x prefix. skbqueue option is just decimal number.
+	OptionSkbmark = "skbmark"
+	// OptionSkbprio All set types support the optional skbinfo extension. This extension allow to store the metainfo (firewall mark, tc class and hardware queue) with every entry and map it to packets by usage of SET netfilter target with --map-set option. skbmark option format: MARK or MARK/MASK, where MARK and MASK are 32bit hex numbers with 0x prefix. If only mark is specified mask 0xffffffff are used. skbprio option has tc class format: MAJOR:MINOR, where major and minor numbers are hex without 0x prefix. skbqueue option is just decimal number.
+	OptionSkbprio = "skbprio"
+	// OptionSkbqueue All set types support the optional skbinfo extension. This extension allow to store the metainfo (firewall mark, tc class and hardware queue) with every entry and map it to packets by usage of SET netfilter target with --map-set option. skbmark option format: MARK or MARK/MASK, where MARK and MASK are 32bit hex numbers with 0x prefix. If only mark is specified mask 0xffffffff are used. skbprio option has tc class format: MAJOR:MINOR, where major and minor numbers are hex without 0x prefix. skbqueue option is just decimal number.
+	OptionSkbqueue = "skbqueue"
+	// OptionHashSize This parameter is valid for the create command of all hash type sets. It defines the initial hash size for the set, default is 1024. The hash size must be a power of two, the kernel automatically rounds up non power of two hash sizes to the first correct value.
+	OptionHashSize = "hashsize"
+	// OptionMaxElem This parameter is valid for the create command of all hash type sets. It does define the maximal number of elements which can be stored in the set, default 65536.
+	OptionMaxElem = "maxelem"
+	// OptionFamilly This parameter is valid for the create command of all hash type sets except for hash:mac. It defines the protocol family of the IP addresses to be stored in the set. The default is inet, i.e IPv4.
+	OptionFamilly = "family"
+	// OptionNoMatch The hash set types which can store net type of data (i.e. hash:*net*) support the optional nomatch option when adding entries. When matching elements in the set, entries marked as nomatch are skipped as if those were not added to the set, which makes possible to build up sets with exceptions. See the example at hash type hash:net below. When elements are tested by ipset, the nomatch flags are taken into account. If one wants to test the existence of an element marked with nomatch in a set, then the flag must be specified too.
+	OptionNoMatch = "nomatch"
+	// OptionForceAdd All hash set types support the optional forceadd parameter when creating a set. When sets created with this option become full the next addition to the set may succeed and evict a random entry from the set.
+	OptionForceAdd = "forceadd"
+)
+
+// IPSet represent ipset sets managed by.
+type IPSet struct {
+	// ipset bianry path.
+	ipSetPath *string
+	// Sets maintainted by ipset.
+	Sets []*Set
+}
+
+// Set reprensent a ipset set entry.
+type Set struct {
+	// ipset parent to get ipSetPath.
+	IPSet *IPSet
+	// set name.
+	Name string
+	// set entries.
+	Entries []*Entry
+	// set created options.
+	Options []string
+}
+
+// Entry of ipset Set.
+type Entry struct {
+	// set parent to get ipSetPath.
+	Set *Set
+	// entry created options.
+	Options []string
+}
+
+// Get ipset binary path or return an error.
+func getIPSetPath() (*string, error) {
+	path, err := exec.LookPath("ipset")
+	if err != nil {
+		return nil, errIpsetNotFound
+	}
+	return &path, nil
+}
+
+// Used to run ipset binary with args and return stdout.
+func (set *IPSet) run(args ...string) (string, error) {
+	var stderr bytes.Buffer
+	var stdout bytes.Buffer
+	cmd := exec.Cmd{
+		Path:   *set.ipSetPath,
+		Args:   append([]string{*set.ipSetPath}, args...),
+		Stderr: &stderr,
+		Stdout: &stdout,
+	}
+
+	if err := cmd.Run(); err != nil {
+		return "", errors.New(stderr.String())
+	}
+
+	return stdout.String(), nil
+}
+
+// Used to run ipset binary with arg and inject stdin buffer and return stdout.
+func (set *IPSet) runWithStdin(stdin *bytes.Buffer, args ...string) (string, error) {
+	var stderr bytes.Buffer
+	var stdout bytes.Buffer
+	cmd := exec.Cmd{
+		Path:   *set.ipSetPath,
+		Args:   append([]string{*set.ipSetPath}, args...),
+		Stderr: &stderr,
+		Stdout: &stdout,
+		Stdin:  stdin,
+	}
+
+	if err := cmd.Run(); err != nil {
+		return "", errors.New(stderr.String())
+	}
+
+	return stdout.String(), nil
+}
+
+// NewIPSet create a new IPSet with ipSetPath initialized.
+func NewIPSet() (*IPSet, error) {
+	ipSetPath, err := getIPSetPath()
+	if err != nil {
+		return nil, err
+	}
+	ipSet := &IPSet{
+		ipSetPath: ipSetPath,
+	}
+	return ipSet, nil
+}
+
+// Create a set identified with setname and specified type. The type may require type specific options. If the -exist option is specified, ipset ignores the error otherwise raised when the same set (setname and create parameters are identical) already exists.
+func (ipSet *IPSet) Create(setName string, createOptions ...string) (*Set, error) {
+	set := &Set{
+		Name:    setName,
+		Options: createOptions,
+	}
+	ipSet.Sets = append(ipSet.Sets, set)
+	_, err := ipSet.run(append([]string{"create", "-exist", set.Name}, createOptions...)...)
+	if err != nil {
+		return nil, err
+	}
+	return set, nil
+}
+
+// Add a given entry to the set. If the -exist option is specified, ipset ignores if the entry already added to the set.
+func (set *Set) Add(addOptions ...string) (*Entry, error) {
+	entry := &Entry{
+		Set:     set,
+		Options: addOptions,
+	}
+	set.Entries = append(set.Entries, entry)
+	_, err := set.IPSet.run(append([]string{"add", "-exist", entry.Set.Name}, addOptions...)...)
+	if err != nil {
+		return nil, err
+	}
+	return entry, nil
+}
+
+// Del an entry from a set. If the -exist option is specified and the entry is not in the set (maybe already expired), then the command is ignored.
+func (entry *Entry) Del() error {
+	_, err := entry.Set.IPSet.run(append([]string{"del", entry.Set.Name}, entry.Options...)...)
+	if err != nil {
+		return err
+	}
+	entry.Set.IPSet.Save()
+	return nil
+}
+
+// Test wether an entry is in a set or not. Exit status number is zero if the tested entry is in the set and nonzero if it is missing from the set.
+func (set *Set) Test(testOptions ...string) (bool, error) {
+	_, err := set.IPSet.run(append([]string{"test", set.Name}, testOptions...)...)
+	if err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
+// Destroy the specified set or all the sets if none is given. If the set has got reference(s), nothing is done and no set destroyed.
+func (set *Set) Destroy() error {
+	_, err := set.IPSet.run("destroy", set.Name)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Destroy the specified set or all the sets if none is given. If the set has got reference(s), nothing is done and no set destroyed.
+func (set *IPSet) Destroy() error {
+	_, err := set.run("destroy")
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Parse ipset save stdout.
+// ex:
+// create KUBE-DST-3YNVZWWGX3UQQ4VQ hash:ip family inet hashsize 1024 maxelem 65536 timeout 0
+// add KUBE-DST-3YNVZWWGX3UQQ4VQ 100.96.1.6 timeout 0
+func parseIPSetSave(ipSet *IPSet, result string) []*Set {
+	sets := make([]*Set, 0)
+	// Save is always in order
+	lines := strings.Split(result, "\n")
+	for _, line := range lines {
+		content := strings.Split(line, " ")
+		if content[0] == "create" {
+			sets = append(sets, &Set{
+				IPSet:   ipSet,
+				Name:    content[1],
+				Options: content[2:],
+			})
+		} else if content[0] == "add" {
+			set := sets[len(sets)-1]
+			set.Entries = append(set.Entries, &Entry{
+				Set:     set,
+				Options: content[2:],
+			})
+		}
+	}
+
+	return sets
+}
+
+// Build ipset restore input
+// ex:
+// create KUBE-DST-3YNVZWWGX3UQQ4VQ hash:ip family inet hashsize 1024 maxelem 65536 timeout 0
+// add KUBE-DST-3YNVZWWGX3UQQ4VQ 100.96.1.6 timeout 0
+func buildIPSetRestore(ipSet *IPSet) string {
+	ipSetRestore := ""
+	for _, set := range ipSet.Sets {
+		ipSetRestore += fmt.Sprintf("create %s %v\n", set.Name, set.Options)
+		for _, entry := range set.Entries {
+			ipSetRestore += fmt.Sprintf("add %s %v\n", set.Name, entry.Options)
+		}
+	}
+	return ipSetRestore
+}
+
+// Save the given set, or all sets if none is given to stdout in a format that restore can read. The option -file can be used to specify a filename instead of stdout.
+func (set *IPSet) Save() error {
+	stdout, err := set.run("save")
+	if err != nil {
+		return err
+	}
+	set.Sets = parseIPSetSave(set, stdout)
+	return nil
+}
+
+// Restore a saved session generated by save. The saved session can be fed from stdin or the option -file can be used to specify a filename instead of stdin. Please note, existing sets and elements are not erased by restore unless specified so in the restore file. All commands are allowed in restore mode except list, help, version, interactive mode and restore itself.
+func (set *IPSet) Restore() error {
+	stdin := bytes.NewBufferString(buildIPSetRestore(set))
+	_, err := set.runWithStdin(stdin, "restore", "-exist")
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Flush all entries from the specified set or flush all sets if none is given.
+func (set *Set) Flush() error {
+	_, err := set.IPSet.run("flush", set.Name)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Flush all entries from the specified set or flush all sets if none is given.
+func (set *IPSet) Flush() error {
+	_, err := set.run("flush")
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Get Set by Name.
+func (ipset *IPSet) Get(setName string) *Set {
+	for _, set := range ipset.Sets {
+		if set.Name == setName {
+			return set
+		}
+	}
+	return nil
+}
+
+// Rename a set. Set identified by SETNAME-TO must not exist.
+func (set *Set) Rename(newName string) error {
+	_, err := set.IPSet.run("rename", set.Name, newName)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Swap the content of two sets, or in another words, exchange the name of two sets. The referred sets must exist and compatible type of sets can be swapped only.
+func (set *Set) Swap(setTo *Set) error {
+	_, err := set.IPSet.run("rename", set.Name, setTo.Name)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Refresh a Set with new entries.
+func (set *Set) Refresh(entries []string, extraOptions ...string) error {
+	tempName := set.Name + "-temp"
+	s := &Set{
+		IPSet:   set.IPSet,
+		Name:    tempName,
+		Options: set.Options,
+	}
+
+	for _, entry := range entries {
+		s.Entries = append(s.Entries, &Entry{
+			Set:     s,
+			Options: append([]string{entry}, extraOptions...),
+		})
+	}
+	set.IPSet.Sets = append(set.IPSet.Sets, s)
+	err := set.IPSet.Restore()
+	if err != nil {
+		return err
+	}
+
+	err = set.Swap(s)
+	if err != nil {
+		return err
+	}
+
+	err = s.Destroy()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

vendor/github.com/janeczku/go-ipset/LICENSE

@@ -1,192 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   Copyright 2014 Docker, Inc.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-   

vendor/github.com/janeczku/go-ipset/README.md

@@ -1,90 +0,0 @@
-# go-ipset #
-
-This library is a simple GoLang wrapper to the IPtables ipset userspace utility.
-It provides an interface to allow Go programs to easily manipulate ipsets.
-It is currently limited to sets of `type hash`.
-
-For ipset command documentation: http://ipset.netfilter.org/ipset.man.html
-
-go-ipset requires ipset kernel module and userspace utility version 6.0 or greater.
-
-## Installation ##
-
-Install go-ipset using the "go get" command:
-
-    go get github.com/janeczku/go-ipset/ipset
-
-Install dependencies:
-
-    go get github.com/Sirupsen/logrus
-    go get github.com/coreos/go-semver/semver
-
-## API Reference ##
-
-[![GoDoc](https://godoc.org/github.com/google/go-github/github?status.svg)](https://godoc.org/github.com/janeczku/go-ipset/ipset)
-
-## Usage ##
-
-```go
-import "github.com/janeczku/go-ipset/ipset
-```
-
-#### Create a new set
-
-Construct a new IPset instance (creating the set on the fly), then use the various methods to manipulate the IPset.
-For example, to create a new ipset "customers" of type `hash:ip` for storing plain IPv4 addresses:
-
-```go
-customers := ipset.New("customers", "hash:ip", &ipset.Params{})
-```
-
-To create a new ipset to store different sized IPv4 network addresses (with /mask).
-
-```go
-trustedNetworks := ipset.New("trusted-networks", "hash:net", &ipset.Params{})
-```
-
-#### Add a single entry to the set
-
-```go
-customers.Add("8.8.2.2")
-```
-
-#### Populate the set with IPv4 addresses (overwriting the previous content)
-
-```go
-ips := []string{"8.8.8.8", "8.8.4.4"}
-customers.Refresh(ips)
-```
-
-#### Remove a single entry from that set:
-
-```go
-customers.Del("8.8.8.8")
-```
-
-#### Configure advanced set options
-
-You can configure advanced options when creating a new set by supplying the parameters in the `ipset.Params` struct.
-
-```go
-type Params struct {
-  HashFamily string
-  HashSize   int
-  MaxElem    int
-  Timeout    int
-}
-```
-See http://ipset.netfilter.org/ipset.man.html for their meaning.
-
-For example, to create a set whose entries will expire after 60 seconds, lets say for temporarily limiting abusive clients:
-
-```go
-abusers := ipset.New("ratelimited", "hash:ip", &ipset.Params{Timeout: 60})
-```
-
-#### List entries of a set
-```go
-// list is []string
-list ipset.List("customers")
-```

vendor/github.com/janeczku/go-ipset/ipset/ipset.go

@@ -1,297 +0,0 @@
-/*
-Copyright 2015 Jan Broer All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Package ipset is a library providing a wrapper to the IPtables ipset userspace utility
-package ipset
-
-import (
-	"errors"
-	"fmt"
-	"os/exec"
-	"regexp"
-	"strconv"
-	"strings"
-
-	"github.com/coreos/go-semver/semver"
-	log "github.com/sirupsen/logrus"
-)
-
-const minIpsetVersion = "6.0.0"
-
-var (
-	ipsetPath            string
-	errIpsetNotFound     = errors.New("Ipset utility not found")
-	errIpsetNotSupported = errors.New("Ipset utility version is not supported, requiring version >= 6.0")
-)
-
-// Params defines optional parameters for creating a new set.
-type Params struct {
-	HashFamily string
-	HashSize   int
-	MaxElem    int
-	Timeout    int
-}
-
-// IPSet implements an Interface to an set.
-type IPSet struct {
-	Name       string
-	HashType   string
-	HashFamily string
-	HashSize   int
-	MaxElem    int
-	Timeout    int
-}
-
-func initCheck() error {
-	if ipsetPath == "" {
-		path, err := exec.LookPath("ipset")
-		if err != nil {
-			return errIpsetNotFound
-		}
-		ipsetPath = path
-		supportedVersion, err := getIpsetSupportedVersion()
-		if err != nil {
-			log.Warnf("Error checking ipset version, assuming version at least 6.0.0: %v", err)
-			supportedVersion = true
-		}
-		if supportedVersion {
-			return nil
-		}
-		return errIpsetNotSupported
-	}
-	return nil
-}
-
-func (s *IPSet) createHashSet(name string) error {
-	/*	out, err := exec.Command("/usr/bin/sudo",
-		ipsetPath, "create", name, s.HashType, "family", s.HashFamily, "hashsize", strconv.Itoa(s.HashSize),
-		"maxelem", strconv.Itoa(s.MaxElem), "timeout", strconv.Itoa(s.Timeout), "-exist").CombinedOutput()*/
-	out, err := exec.Command(ipsetPath, "create", name, s.HashType, "family", s.HashFamily, "hashsize", strconv.Itoa(s.HashSize),
-		"maxelem", strconv.Itoa(s.MaxElem), "timeout", strconv.Itoa(s.Timeout), "-exist").CombinedOutput()
-	if err != nil {
-		return fmt.Errorf("error creating ipset %s with type %s: %v (%s)", name, s.HashType, err, out)
-	}
-	out, err = exec.Command(ipsetPath, "flush", name).CombinedOutput()
-	if err != nil {
-		return fmt.Errorf("error flushing ipset %s: %v (%s)", name, err, out)
-	}
-	return nil
-}
-
-// New creates a new set and returns an Interface to it.
-// Example:
-// 	testIpset := ipset.New("test", "hash:ip", &ipset.Params{})
-func New(name string, hashtype string, p *Params) (*IPSet, error) {
-	// Using the ipset utilities default values here
-	if p.HashSize == 0 {
-		p.HashSize = 1024
-	}
-
-	if p.MaxElem == 0 {
-		p.MaxElem = 65536
-	}
-
-	if p.HashFamily == "" {
-		p.HashFamily = "inet"
-	}
-
-	// Check if hashtype is a type of hash
-	if !strings.HasPrefix(hashtype, "hash:") {
-		return nil, fmt.Errorf("not a hash type: %s", hashtype)
-	}
-
-	if err := initCheck(); err != nil {
-		return nil, err
-	}
-
-	s := IPSet{name, hashtype, p.HashFamily, p.HashSize, p.MaxElem, p.Timeout}
-	err := s.createHashSet(name)
-	if err != nil {
-		return nil, err
-	}
-	return &s, nil
-}
-
-// Refresh is used to to overwrite the set with the specified entries.
-// The ipset is updated on the fly by hot swapping it with a temporary set.
-func (s *IPSet) Refresh(entries []string) error {
-	tempName := s.Name + "-temp"
-	err := s.createHashSet(tempName)
-	if err != nil {
-		return err
-	}
-	for _, entry := range entries {
-		out, err := exec.Command(ipsetPath, "add", tempName, entry, "-exist").CombinedOutput()
-		if err != nil {
-			log.Errorf("error adding entry %s to set %s: %v (%s)", entry, tempName, err, out)
-		}
-	}
-	err = Swap(tempName, s.Name)
-	if err != nil {
-		return err
-	}
-	err = destroyIPSet(tempName)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-// Test is used to check whether the specified entry is in the set or not.
-func (s *IPSet) Test(entry string) (bool, error) {
-	out, err := exec.Command(ipsetPath, "test", s.Name, entry).CombinedOutput()
-	if err == nil {
-		reg, e := regexp.Compile("NOT")
-		if e == nil && reg.MatchString(string(out)) {
-			return false, nil
-		} else if e == nil {
-			return true, nil
-		} else {
-			return false, fmt.Errorf("error testing entry %s: %v", entry, e)
-		}
-	} else {
-		return false, fmt.Errorf("error testing entry %s: %v (%s)", entry, err, out)
-	}
-}
-
-// Add is used to add the specified entry to the set.
-// A timeout of 0 means that the entry will be stored permanently in the set.
-func (s *IPSet) Add(entry string, timeout int) error {
-	out, err := exec.Command(ipsetPath, "add", s.Name, entry, "timeout", strconv.Itoa(timeout), "-exist").CombinedOutput()
-	if err != nil {
-		return fmt.Errorf("error adding entry %s: %v (%s)", entry, err, out)
-	}
-	return nil
-}
-
-// AddOption is used to add the specified entry to the set.
-// A timeout of 0 means that the entry will be stored permanently in the set.
-func (s *IPSet) AddOption(entry string, option string, timeout int) error {
-	out, err := exec.Command(ipsetPath, "add", s.Name, entry, option, "timeout", strconv.Itoa(timeout), "-exist").CombinedOutput()
-	if err != nil {
-		return fmt.Errorf("error adding entry %s with option %s : %v (%s)", entry, option, err, out)
-	}
-	return nil
-}
-
-// Del is used to delete the specified entry from the set.
-func (s *IPSet) Del(entry string) error {
-	out, err := exec.Command(ipsetPath, "del", s.Name, entry, "-exist").CombinedOutput()
-	if err != nil {
-		return fmt.Errorf("error deleting entry %s: %v (%s)", entry, err, out)
-	}
-	return nil
-}
-
-// Flush is used to flush all entries in the set.
-func (s *IPSet) Flush() error {
-	out, err := exec.Command(ipsetPath, "flush", s.Name).CombinedOutput()
-	if err != nil {
-		return fmt.Errorf("error flushing set %s: %v (%s)", s.Name, err, out)
-	}
-	return nil
-}
-
-// List is used to show the contents of a set
-func (s *IPSet) List() ([]string, error) {
-	out, err := exec.Command(ipsetPath, "list", s.Name).CombinedOutput()
-	if err != nil {
-		return []string{}, fmt.Errorf("error listing set %s: %v (%s)", s.Name, err, out)
-	}
-	r := regexp.MustCompile("(?m)^(.*\n)*Members:\n")
-	list := r.ReplaceAllString(string(out[:]), "")
-	return strings.Split(list, "\n"), nil
-}
-
-// Destroy is used to destroy the set.
-func (s *IPSet) Destroy() error {
-	out, err := exec.Command(ipsetPath, "destroy", s.Name).CombinedOutput()
-	if err != nil {
-		return fmt.Errorf("error destroying set %s: %v (%s)", s.Name, err, out)
-	}
-	return nil
-}
-
-// DestroyAll is used to destroy the set.
-func DestroyAll() error {
-	initCheck()
-	out, err := exec.Command(ipsetPath, "destroy").CombinedOutput()
-	if err != nil {
-		return fmt.Errorf("error destroying set %s (%s)", err, out)
-	}
-	return nil
-}
-
-// Swap is used to hot swap two sets on-the-fly. Use with names of existing sets of the same type.
-func Swap(from, to string) error {
-	out, err := exec.Command(ipsetPath, "swap", from, to).Output()
-	if err != nil {
-		return fmt.Errorf("error swapping ipset %s to %s: %v (%s)", from, to, err, out)
-	}
-	return nil
-}
-
-func destroyIPSet(name string) error {
-	out, err := exec.Command(ipsetPath, "destroy", name).Output()
-	if err != nil {
-		return fmt.Errorf("error destroying ipset %s: %v (%s)", name, err, out)
-	}
-	return nil
-}
-
-func destroyAll() error {
-	out, err := exec.Command(ipsetPath, "destroy").Output()
-	if err != nil {
-		return fmt.Errorf("error destroying all ipsetz %s (%s)", err, out)
-	}
-	return nil
-}
-
-func getIpsetSupportedVersion() (bool, error) {
-	minVersion, err := semver.NewVersion(minIpsetVersion)
-	if err != nil {
-		return false, err
-	}
-	// Returns "vX.Y".
-	vstring, err := getIpsetVersionString()
-	if err != nil {
-		return false, err
-	}
-	// Make a dotted-tri format version string
-	vstring = vstring + ".0"
-	// Make a semver of the part after the v in "vX.X.X".
-	version, err := semver.NewVersion(vstring[1:])
-	if err != nil {
-		return false, err
-	}
-	if version.LessThan(*minVersion) {
-		return false, nil
-	}
-	return true, nil
-}
-
-func getIpsetVersionString() (string, error) {
-	bytes, err := exec.Command(ipsetPath, "--version").CombinedOutput()
-	if err != nil {
-		return "", err
-	}
-	versionMatcher := regexp.MustCompile("v[0-9]+\\.[0-9]+")
-	match := versionMatcher.FindStringSubmatch(string(bytes))
-	if match == nil {
-		return "", fmt.Errorf("no ipset version found in string: %s", bytes)
-	}
-	return match[0], nil
-}