From 3dda813d544a29e004242a3affcfec21fb1d18d7 Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Fri, 13 Jun 2025 00:13:06 +0200 Subject: [PATCH] IMPORT: eb32/64: optimize insert for modern CPUs Similar to previous patches, let's improve the insert() descent loop to avoid discovering mandatory data too late. The change here is even simpler than previous ones, a prefetch was installed and troot is calculated before last instruction in a speculative way. This was enough to gain +50% insertion rate on random data. This is ebtree commit e893f8cc4d44b10f406b9d1d78bd4a9bd9183ccf. --- include/import/eb32tree.h | 16 ++++++++++++---- include/import/eb64tree.h | 16 ++++++++++++---- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/include/import/eb32tree.h b/include/import/eb32tree.h index 4c7a3a649..292a10e39 100644 --- a/include/import/eb32tree.h +++ b/include/import/eb32tree.h @@ -270,6 +270,10 @@ __eb32_insert(struct eb_root *root, struct eb32_node *new) { /* OK we're walking down this link */ old = container_of(eb_untag(troot, EB_NODE), struct eb32_node, node.branches); + + __builtin_prefetch(old->node.branches.b[0], 0); + __builtin_prefetch(old->node.branches.b[1], 0); + old_node_bit = old->node.bit; /* Stop going down when we don't have common bits anymore. We @@ -289,9 +293,9 @@ __eb32_insert(struct eb_root *root, struct eb32_node *new) { } /* walk down */ - root = &old->node.branches; side = (newkey >> old_node_bit) & EB_NODE_BRANCH_MASK; - troot = root->b[side]; + troot = side ? old->node.branches.b[1] : old->node.branches.b[0]; + root = &old->node.branches; } new_left = eb_dotag(&new->node.branches, EB_LEFT); @@ -403,6 +407,10 @@ __eb32i_insert(struct eb_root *root, struct eb32_node *new) { /* OK we're walking down this link */ old = container_of(eb_untag(troot, EB_NODE), struct eb32_node, node.branches); + + __builtin_prefetch(old->node.branches.b[0], 0); + __builtin_prefetch(old->node.branches.b[1], 0); + old_node_bit = old->node.bit; /* Stop going down when we don't have common bits anymore. We @@ -422,9 +430,9 @@ __eb32i_insert(struct eb_root *root, struct eb32_node *new) { } /* walk down */ - root = &old->node.branches; side = (newkey >> old_node_bit) & EB_NODE_BRANCH_MASK; - troot = root->b[side]; + troot = side ? old->node.branches.b[1] : old->node.branches.b[0]; + root = &old->node.branches; } new_left = eb_dotag(&new->node.branches, EB_LEFT); diff --git a/include/import/eb64tree.h b/include/import/eb64tree.h index 7e3a0f0f3..5de45dd6c 100644 --- a/include/import/eb64tree.h +++ b/include/import/eb64tree.h @@ -316,6 +316,10 @@ __eb64_insert(struct eb_root *root, struct eb64_node *new) { /* OK we're walking down this link */ old = container_of(eb_untag(troot, EB_NODE), struct eb64_node, node.branches); + + __builtin_prefetch(old->node.branches.b[0], 0); + __builtin_prefetch(old->node.branches.b[1], 0); + old_node_bit = old->node.bit; /* Stop going down when we don't have common bits anymore. We @@ -360,7 +364,6 @@ __eb64_insert(struct eb_root *root, struct eb64_node *new) { } /* walk down */ - root = &old->node.branches; if (sizeof(long) >= 8) { side = newkey >> old_node_bit; @@ -376,7 +379,8 @@ __eb64_insert(struct eb_root *root, struct eb64_node *new) { } } side &= EB_NODE_BRANCH_MASK; - troot = root->b[side]; + troot = side ? old->node.branches.b[1] : old->node.branches.b[0]; + root = &old->node.branches; } /* Ok, now we are inserting between and . 's @@ -498,6 +502,10 @@ __eb64i_insert(struct eb_root *root, struct eb64_node *new) { /* OK we're walking down this link */ old = container_of(eb_untag(troot, EB_NODE), struct eb64_node, node.branches); + + __builtin_prefetch(old->node.branches.b[0], 0); + __builtin_prefetch(old->node.branches.b[1], 0); + old_node_bit = old->node.bit; /* Stop going down when we don't have common bits anymore. We @@ -542,7 +550,6 @@ __eb64i_insert(struct eb_root *root, struct eb64_node *new) { } /* walk down */ - root = &old->node.branches; if (sizeof(long) >= 8) { side = newkey >> old_node_bit; @@ -558,7 +565,8 @@ __eb64i_insert(struct eb_root *root, struct eb64_node *new) { } } side &= EB_NODE_BRANCH_MASK; - troot = root->b[side]; + troot = side ? old->node.branches.b[1] : old->node.branches.b[0]; + root = &old->node.branches; } /* Ok, now we are inserting between and . 's