mirrors/haproxy
1
0
Fork 0
mirror of https://git.haproxy.org/git/haproxy.git/ synced 2025-09-20 13:21:29 +02:00
Code Issues Projects Releases Wiki Activity

IMPORT: eb32/64: optimize insert for modern CPUs

Browse Source 
Similar to previous patches, let's improve the insert() descent loop to
avoid discovering mandatory data too late. The change here is even
simpler than previous ones, a prefetch was installed and troot is
calculated before last instruction in a speculative way. This was enough
to gain +50% insertion rate on random data.

This is ebtree commit e893f8cc4d44b10f406b9d1d78bd4a9bd9183ccf.
This commit is contained in:
Willy Tarreau 2025-06-13 00:13:06 +02:00
parent 61654c07bd
commit 3dda813d54
2 changed files with 24 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
include/import/eb32tree.h
View File

@ -270,6 +270,10 @@ __eb32_insert(struct eb_root *root, struct eb32_node *new) {
/* OK we're walking down this link */ /* OK we're walking down this link */
old = container_of(eb_untag(troot, EB_NODE), old = container_of(eb_untag(troot, EB_NODE),
struct eb32_node, node.branches); struct eb32_node, node.branches);
__builtin_prefetch(old->node.branches.b[0], 0);
__builtin_prefetch(old->node.branches.b[1], 0);
old_node_bit = old->node.bit; old_node_bit = old->node.bit;
/* Stop going down when we don't have common bits anymore. We /* Stop going down when we don't have common bits anymore. We
@ -289,9 +293,9 @@ __eb32_insert(struct eb_root *root, struct eb32_node *new) {
} }
/* walk down */ /* walk down */
root = &old->node.branches;
side = (newkey >> old_node_bit) & EB_NODE_BRANCH_MASK; side = (newkey >> old_node_bit) & EB_NODE_BRANCH_MASK;
troot = root->b[side]; troot = side ? old->node.branches.b[1] : old->node.branches.b[0];
root = &old->node.branches;
} }
new_left = eb_dotag(&new->node.branches, EB_LEFT); new_left = eb_dotag(&new->node.branches, EB_LEFT);
@ -403,6 +407,10 @@ __eb32i_insert(struct eb_root *root, struct eb32_node *new) {
/* OK we're walking down this link */ /* OK we're walking down this link */
old = container_of(eb_untag(troot, EB_NODE), old = container_of(eb_untag(troot, EB_NODE),
struct eb32_node, node.branches); struct eb32_node, node.branches);
__builtin_prefetch(old->node.branches.b[0], 0);
__builtin_prefetch(old->node.branches.b[1], 0);
old_node_bit = old->node.bit; old_node_bit = old->node.bit;
/* Stop going down when we don't have common bits anymore. We /* Stop going down when we don't have common bits anymore. We
@ -422,9 +430,9 @@ __eb32i_insert(struct eb_root *root, struct eb32_node *new) {
} }
/* walk down */ /* walk down */
root = &old->node.branches;
side = (newkey >> old_node_bit) & EB_NODE_BRANCH_MASK; side = (newkey >> old_node_bit) & EB_NODE_BRANCH_MASK;
troot = root->b[side]; troot = side ? old->node.branches.b[1] : old->node.branches.b[0];
root = &old->node.branches;
} }
new_left = eb_dotag(&new->node.branches, EB_LEFT); new_left = eb_dotag(&new->node.branches, EB_LEFT);

16
include/import/eb64tree.h
View File

@ -316,6 +316,10 @@ __eb64_insert(struct eb_root *root, struct eb64_node *new) {
/* OK we're walking down this link */ /* OK we're walking down this link */
old = container_of(eb_untag(troot, EB_NODE), old = container_of(eb_untag(troot, EB_NODE),
struct eb64_node, node.branches); struct eb64_node, node.branches);
__builtin_prefetch(old->node.branches.b[0], 0);
__builtin_prefetch(old->node.branches.b[1], 0);
old_node_bit = old->node.bit; old_node_bit = old->node.bit;
/* Stop going down when we don't have common bits anymore. We /* Stop going down when we don't have common bits anymore. We
@ -360,7 +364,6 @@ __eb64_insert(struct eb_root *root, struct eb64_node *new) {
} }
/* walk down */ /* walk down */
root = &old->node.branches;
if (sizeof(long) >= 8) { if (sizeof(long) >= 8) {
side = newkey >> old_node_bit; side = newkey >> old_node_bit;
@ -376,7 +379,8 @@ __eb64_insert(struct eb_root *root, struct eb64_node *new) {
} }
} }
side &= EB_NODE_BRANCH_MASK; side &= EB_NODE_BRANCH_MASK;
troot = root->b[side]; troot = side ? old->node.branches.b[1] : old->node.branches.b[0];
root = &old->node.branches;
} }
/* Ok, now we are inserting <new> between <root> and <old>. <old>'s /* Ok, now we are inserting <new> between <root> and <old>. <old>'s
@ -498,6 +502,10 @@ __eb64i_insert(struct eb_root *root, struct eb64_node *new) {
/* OK we're walking down this link */ /* OK we're walking down this link */
old = container_of(eb_untag(troot, EB_NODE), old = container_of(eb_untag(troot, EB_NODE),
struct eb64_node, node.branches); struct eb64_node, node.branches);
__builtin_prefetch(old->node.branches.b[0], 0);
__builtin_prefetch(old->node.branches.b[1], 0);
old_node_bit = old->node.bit; old_node_bit = old->node.bit;
/* Stop going down when we don't have common bits anymore. We /* Stop going down when we don't have common bits anymore. We
@ -542,7 +550,6 @@ __eb64i_insert(struct eb_root *root, struct eb64_node *new) {
} }
/* walk down */ /* walk down */
root = &old->node.branches;
if (sizeof(long) >= 8) { if (sizeof(long) >= 8) {
side = newkey >> old_node_bit; side = newkey >> old_node_bit;
@ -558,7 +565,8 @@ __eb64i_insert(struct eb_root *root, struct eb64_node *new) {
} }
} }
side &= EB_NODE_BRANCH_MASK; side &= EB_NODE_BRANCH_MASK;
troot = root->b[side]; troot = side ? old->node.branches.b[1] : old->node.branches.b[0];
root = &old->node.branches;
} }
/* Ok, now we are inserting <new> between <root> and <old>. <old>'s /* Ok, now we are inserting <new> between <root> and <old>. <old>'s