diff --git a/include/import/eb32tree.h b/include/import/eb32tree.h index 1c03fc1ed..be45e19aa 100644 --- a/include/import/eb32tree.h +++ b/include/import/eb32tree.h @@ -119,7 +119,7 @@ static forceinline struct eb32_node *__eb32_lookup(struct eb_root *root, u32 x) { struct eb32_node *node; eb_troot_t *troot; - u32 y; + u32 y, z; int node_bit; troot = root->b[EB_LEFT]; @@ -137,9 +137,15 @@ static forceinline struct eb32_node *__eb32_lookup(struct eb_root *root, u32 x) } node = container_of(eb_untag(troot, EB_NODE), struct eb32_node, node.branches); - node_bit = node->node.bit; + __builtin_prefetch(node->node.branches.b[0], 0); + __builtin_prefetch(node->node.branches.b[1], 0); + + node_bit = node->node.bit; y = node->key ^ x; + z = 1U << (node_bit & 31); + troot = (x & z) ? node->node.branches.b[1] : node->node.branches.b[0]; + if (!y) { /* Either we found the node which holds the key, or * we have a dup tree. In the later case, we have to @@ -157,8 +163,6 @@ static forceinline struct eb32_node *__eb32_lookup(struct eb_root *root, u32 x) if ((y >> node_bit) >= EB_NODE_BRANCHES) return NULL; /* no more common bits */ - - troot = node->node.branches.b[(x >> node_bit) & EB_NODE_BRANCH_MASK]; } } @@ -171,7 +175,7 @@ static forceinline struct eb32_node *__eb32i_lookup(struct eb_root *root, s32 x) struct eb32_node *node; eb_troot_t *troot; u32 key = x ^ 0x80000000; - u32 y; + u32 y, z; int node_bit; troot = root->b[EB_LEFT]; @@ -189,9 +193,15 @@ static forceinline struct eb32_node *__eb32i_lookup(struct eb_root *root, s32 x) } node = container_of(eb_untag(troot, EB_NODE), struct eb32_node, node.branches); - node_bit = node->node.bit; + __builtin_prefetch(node->node.branches.b[0], 0); + __builtin_prefetch(node->node.branches.b[1], 0); + + node_bit = node->node.bit; y = node->key ^ x; + z = 1U << (node_bit & 31); + troot = (key & z) ? node->node.branches.b[1] : node->node.branches.b[0]; + if (!y) { /* Either we found the node which holds the key, or * we have a dup tree. In the later case, we have to @@ -209,8 +219,6 @@ static forceinline struct eb32_node *__eb32i_lookup(struct eb_root *root, s32 x) if ((y >> node_bit) >= EB_NODE_BRANCHES) return NULL; /* no more common bits */ - - troot = node->node.branches.b[(key >> node_bit) & EB_NODE_BRANCH_MASK]; } } diff --git a/include/import/eb64tree.h b/include/import/eb64tree.h index d6e5db49c..75c13c367 100644 --- a/include/import/eb64tree.h +++ b/include/import/eb64tree.h @@ -119,7 +119,7 @@ static forceinline struct eb64_node *__eb64_lookup(struct eb_root *root, u64 x) { struct eb64_node *node; eb_troot_t *troot; - u64 y; + u64 y, z; troot = root->b[EB_LEFT]; if (unlikely(troot == NULL)) @@ -137,7 +137,13 @@ static forceinline struct eb64_node *__eb64_lookup(struct eb_root *root, u64 x) node = container_of(eb_untag(troot, EB_NODE), struct eb64_node, node.branches); + __builtin_prefetch(node->node.branches.b[0], 0); + __builtin_prefetch(node->node.branches.b[1], 0); + y = node->key ^ x; + z = 1ULL << (node->node.bit & 63); + troot = (x & z) ? node->node.branches.b[1] : node->node.branches.b[0]; + if (!y) { /* Either we found the node which holds the key, or * we have a dup tree. In the later case, we have to @@ -155,8 +161,6 @@ static forceinline struct eb64_node *__eb64_lookup(struct eb_root *root, u64 x) if ((y >> node->node.bit) >= EB_NODE_BRANCHES) return NULL; /* no more common bits */ - - troot = node->node.branches.b[(x >> node->node.bit) & EB_NODE_BRANCH_MASK]; } } @@ -169,7 +173,7 @@ static forceinline struct eb64_node *__eb64i_lookup(struct eb_root *root, s64 x) struct eb64_node *node; eb_troot_t *troot; u64 key = x ^ (1ULL << 63); - u64 y; + u64 y, z; troot = root->b[EB_LEFT]; if (unlikely(troot == NULL)) @@ -187,7 +191,13 @@ static forceinline struct eb64_node *__eb64i_lookup(struct eb_root *root, s64 x) node = container_of(eb_untag(troot, EB_NODE), struct eb64_node, node.branches); + __builtin_prefetch(node->node.branches.b[0], 0); + __builtin_prefetch(node->node.branches.b[1], 0); + y = node->key ^ x; + z = 1ULL << (node->node.bit & 63); + troot = (key & z) ? node->node.branches.b[1] : node->node.branches.b[0]; + if (!y) { /* Either we found the node which holds the key, or * we have a dup tree. In the later case, we have to @@ -205,8 +215,6 @@ static forceinline struct eb64_node *__eb64i_lookup(struct eb_root *root, s64 x) if ((y >> node->node.bit) >= EB_NODE_BRANCHES) return NULL; /* no more common bits */ - - troot = node->node.branches.b[(key >> node->node.bit) & EB_NODE_BRANCH_MASK]; } } diff --git a/src/eb32tree.c b/src/eb32tree.c index 38ddab00d..19af56e1a 100644 --- a/src/eb32tree.c +++ b/src/eb32tree.c @@ -50,6 +50,7 @@ struct eb32_node *eb32_lookup_le(struct eb_root *root, u32 x) { struct eb32_node *node; eb_troot_t *troot; + u32 y, z; troot = root->b[EB_LEFT]; if (unlikely(troot == NULL)) @@ -72,6 +73,13 @@ struct eb32_node *eb32_lookup_le(struct eb_root *root, u32 x) node = container_of(eb_untag(troot, EB_NODE), struct eb32_node, node.branches); + __builtin_prefetch(node->node.branches.b[0], 0); + __builtin_prefetch(node->node.branches.b[1], 0); + + y = node->key; + z = 1U << (node->node.bit & 31); + troot = (x & z) ? node->node.branches.b[1] : node->node.branches.b[0]; + if (node->node.bit < 0) { /* We're at the top of a dup tree. Either we got a * matching value and we return the rightmost node, or @@ -93,7 +101,7 @@ struct eb32_node *eb32_lookup_le(struct eb_root *root, u32 x) break; } - if (((x ^ node->key) >> node->node.bit) >= EB_NODE_BRANCHES) { + if ((x ^ y) & -(z << 1)) { /* No more common bits at all. Either this node is too * small and we need to get its highest value, or it is * too large, and we need to get the prev value. @@ -109,7 +117,6 @@ struct eb32_node *eb32_lookup_le(struct eb_root *root, u32 x) troot = node->node.node_p; break; } - troot = node->node.branches.b[(x >> node->node.bit) & EB_NODE_BRANCH_MASK]; } /* If we get here, it means we want to report previous node before the @@ -138,6 +145,7 @@ struct eb32_node *eb32_lookup_ge(struct eb_root *root, u32 x) { struct eb32_node *node; eb_troot_t *troot; + u32 y, z; troot = root->b[EB_LEFT]; if (unlikely(troot == NULL)) @@ -160,6 +168,13 @@ struct eb32_node *eb32_lookup_ge(struct eb_root *root, u32 x) node = container_of(eb_untag(troot, EB_NODE), struct eb32_node, node.branches); + __builtin_prefetch(node->node.branches.b[0], 0); + __builtin_prefetch(node->node.branches.b[1], 0); + + y = node->key; + z = 1U << (node->node.bit & 31); + troot = (x & z) ? node->node.branches.b[1] : node->node.branches.b[0]; + if (node->node.bit < 0) { /* We're at the top of a dup tree. Either we got a * matching value and we return the leftmost node, or @@ -181,7 +196,7 @@ struct eb32_node *eb32_lookup_ge(struct eb_root *root, u32 x) break; } - if (((x ^ node->key) >> node->node.bit) >= EB_NODE_BRANCHES) { + if ((x ^ y) & -(z << 1)) { /* No more common bits at all. Either this node is too * large and we need to get its lowest value, or it is too * small, and we need to get the next value. @@ -197,7 +212,6 @@ struct eb32_node *eb32_lookup_ge(struct eb_root *root, u32 x) troot = node->node.node_p; break; } - troot = node->node.branches.b[(x >> node->node.bit) & EB_NODE_BRANCH_MASK]; } /* If we get here, it means we want to report next node after the diff --git a/src/eb64tree.c b/src/eb64tree.c index b908d4db2..e5f6322bd 100644 --- a/src/eb64tree.c +++ b/src/eb64tree.c @@ -50,6 +50,7 @@ struct eb64_node *eb64_lookup_le(struct eb_root *root, u64 x) { struct eb64_node *node; eb_troot_t *troot; + u64 y, z; troot = root->b[EB_LEFT]; if (unlikely(troot == NULL)) @@ -72,6 +73,13 @@ struct eb64_node *eb64_lookup_le(struct eb_root *root, u64 x) node = container_of(eb_untag(troot, EB_NODE), struct eb64_node, node.branches); + __builtin_prefetch(node->node.branches.b[0], 0); + __builtin_prefetch(node->node.branches.b[1], 0); + + y = node->key; + z = 1ULL << (node->node.bit & 63); + troot = (x & z) ? node->node.branches.b[1] : node->node.branches.b[0]; + if (node->node.bit < 0) { /* We're at the top of a dup tree. Either we got a * matching value and we return the rightmost node, or @@ -93,7 +101,7 @@ struct eb64_node *eb64_lookup_le(struct eb_root *root, u64 x) break; } - if (((x ^ node->key) >> node->node.bit) >= EB_NODE_BRANCHES) { + if ((x ^ y) & -(z << 1)) { /* No more common bits at all. Either this node is too * small and we need to get its highest value, or it is * too large, and we need to get the prev value. @@ -109,7 +117,6 @@ struct eb64_node *eb64_lookup_le(struct eb_root *root, u64 x) troot = node->node.node_p; break; } - troot = node->node.branches.b[(x >> node->node.bit) & EB_NODE_BRANCH_MASK]; } /* If we get here, it means we want to report previous node before the @@ -138,6 +145,7 @@ struct eb64_node *eb64_lookup_ge(struct eb_root *root, u64 x) { struct eb64_node *node; eb_troot_t *troot; + u64 y, z; troot = root->b[EB_LEFT]; if (unlikely(troot == NULL)) @@ -160,6 +168,13 @@ struct eb64_node *eb64_lookup_ge(struct eb_root *root, u64 x) node = container_of(eb_untag(troot, EB_NODE), struct eb64_node, node.branches); + __builtin_prefetch(node->node.branches.b[0], 0); + __builtin_prefetch(node->node.branches.b[1], 0); + + y = node->key; + z = 1ULL << (node->node.bit & 63); + troot = (x & z) ? node->node.branches.b[1] : node->node.branches.b[0]; + if (node->node.bit < 0) { /* We're at the top of a dup tree. Either we got a * matching value and we return the leftmost node, or @@ -181,7 +196,7 @@ struct eb64_node *eb64_lookup_ge(struct eb_root *root, u64 x) break; } - if (((x ^ node->key) >> node->node.bit) >= EB_NODE_BRANCHES) { + if ((x ^ y) & -(z << 1)) { /* No more common bits at all. Either this node is too * large and we need to get its lowest value, or it is too * small, and we need to get the next value. @@ -197,7 +212,6 @@ struct eb64_node *eb64_lookup_ge(struct eb_root *root, u64 x) troot = node->node.node_p; break; } - troot = node->node.branches.b[(x >> node->node.bit) & EB_NODE_BRANCH_MASK]; } /* If we get here, it means we want to report next node after the