diff --git a/pkg/controllers/netpol/network_policy_controller.go b/pkg/controllers/netpol/network_policy_controller.go
index e7ab15a6..57f44206 100644
--- a/pkg/controllers/netpol/network_policy_controller.go
+++ b/pkg/controllers/netpol/network_policy_controller.go
@@ -684,17 +684,28 @@ func (npc *NetworkPolicyController) cleanupStaleIPSets(activePolicyIPSets map[st
 		}()
 	}
 
-	for _, ipsets := range npc.ipSetHandlers {
+	for ipFamily, ipsets := range npc.ipSetHandlers {
 		cleanupPolicyIPSets := make([]*utils.Set, 0)
 
 		if err := ipsets.Save(); err != nil {
 			klog.Fatalf("failed to initialize ipsets command executor due to %s", err.Error())
 		}
-		for _, set := range ipsets.Sets() {
-			if strings.HasPrefix(set.Name, kubeSourceIPSetPrefix) ||
-				strings.HasPrefix(set.Name, kubeDestinationIPSetPrefix) {
-				if _, ok := activePolicyIPSets[set.Name]; !ok {
-					cleanupPolicyIPSets = append(cleanupPolicyIPSets, set)
+		if ipFamily == v1core.IPv6Protocol {
+			for _, set := range ipsets.Sets() {
+				if strings.HasPrefix(set.Name, fmt.Sprintf("%s:%s", utils.FamillyInet6, kubeSourceIPSetPrefix)) ||
+					strings.HasPrefix(set.Name, fmt.Sprintf("%s:%s", utils.FamillyInet6, kubeDestinationIPSetPrefix)) {
+					if _, ok := activePolicyIPSets[set.Name]; !ok {
+						cleanupPolicyIPSets = append(cleanupPolicyIPSets, set)
+					}
+				}
+			}
+		} else {
+			for _, set := range ipsets.Sets() {
+				if strings.HasPrefix(set.Name, kubeSourceIPSetPrefix) ||
+					strings.HasPrefix(set.Name, kubeDestinationIPSetPrefix) {
+					if _, ok := activePolicyIPSets[set.Name]; !ok {
+						cleanupPolicyIPSets = append(cleanupPolicyIPSets, set)
+					}
 				}
 			}
 		}
diff --git a/pkg/controllers/netpol/policy.go b/pkg/controllers/netpol/policy.go
index 73797916..6952cb7b 100644
--- a/pkg/controllers/netpol/policy.go
+++ b/pkg/controllers/netpol/policy.go
@@ -474,10 +474,10 @@ func (npc *NetworkPolicyController) appendRuleToPolicyChain(policyChainName, com
 		args = append(args, "-m", "comment", "--comment", "\""+comment+"\"")
 	}
 	if srcIPSetName != "" {
-		args = append(args, "-m", "set", "--match-set", srcIPSetName, "src")
+		args = append(args, "-m", "set", "--match-set", npc.ipSetHandlers[ipFamily].Name(srcIPSetName), "src")
 	}
 	if dstIPSetName != "" {
-		args = append(args, "-m", "set", "--match-set", dstIPSetName, "dst")
+		args = append(args, "-m", "set", "--match-set", npc.ipSetHandlers[ipFamily].Name(dstIPSetName), "dst")
 	}
 	if protocol != "" {
 		args = append(args, "-p", protocol)