From 75cf17ee30006c08589b46e69966c9e1d9ab4975 Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Fri, 29 Aug 2008 15:48:49 +0200 Subject: [PATCH] [OPTIM] force inlining of large functions with gcc >= 3 GCC 3 and above do not inline large functions, which is a problem with ebtree where most core functions are inlined. This simple patch has both reduced code size and increased speed. It should be back-ported to ebtree. --- include/common/config.h | 11 +++++++++++ include/common/eb32tree.h | 10 +++++----- include/common/eb64tree.h | 10 +++++----- include/common/ebpttree.h | 6 +++--- include/common/ebtree.h | 15 +++++++++++++-- 5 files changed, 37 insertions(+), 15 deletions(-) diff --git a/include/common/config.h b/include/common/config.h index 74bee477e..5efa7cbb5 100644 --- a/include/common/config.h +++ b/include/common/config.h @@ -83,4 +83,15 @@ #define REGPRM3 #endif +/* By default, gcc does not inline large chunks of code, but we want it to + * respect our choices. + */ +#if !defined(forceinline) +#if __GNUC__ < 3 +#define forceinline inline +#else +#define forceinline inline __attribute__((always_inline)) +#endif +#endif + #endif /* _COMMON_CONFIG_H */ diff --git a/include/common/eb32tree.h b/include/common/eb32tree.h index 6a39595b5..b5c88eeab 100644 --- a/include/common/eb32tree.h +++ b/include/common/eb32tree.h @@ -109,7 +109,7 @@ REGPRM2 struct eb32_node *eb32i_insert(struct eb_root *root, struct eb32_node *n */ /* Delete node from the tree if it was linked in. Mark the node unused. */ -static inline void __eb32_delete(struct eb32_node *eb32) +static forceinline void __eb32_delete(struct eb32_node *eb32) { __eb_delete(&eb32->node); } @@ -118,7 +118,7 @@ static inline void __eb32_delete(struct eb32_node *eb32) * Find the first occurence of a key in the tree . If none can be * found, return NULL. */ -static inline struct eb32_node *__eb32_lookup(struct eb_root *root, u32 x) +static forceinline struct eb32_node *__eb32_lookup(struct eb_root *root, u32 x) { struct eb32_node *node; eb_troot_t *troot; @@ -162,7 +162,7 @@ static inline struct eb32_node *__eb32_lookup(struct eb_root *root, u32 x) * Find the first occurence of a signed key in the tree . If none can * be found, return NULL. */ -static inline struct eb32_node *__eb32i_lookup(struct eb_root *root, s32 x) +static forceinline struct eb32_node *__eb32i_lookup(struct eb_root *root, s32 x) { struct eb32_node *node; eb_troot_t *troot; @@ -207,7 +207,7 @@ static inline struct eb32_node *__eb32i_lookup(struct eb_root *root, s32 x) * Only new->key needs be set with the key. The eb32_node is returned. * If root->b[EB_RGHT]==1, the tree may only contain unique keys. */ -static inline struct eb32_node * +static forceinline struct eb32_node * __eb32_insert(struct eb_root *root, struct eb32_node *new) { struct eb32_node *old; unsigned int side; @@ -371,7 +371,7 @@ __eb32_insert(struct eb_root *root, struct eb32_node *new) { * signed keys. Only new->key needs be set with the key. The eb32_node * is returned. If root->b[EB_RGHT]==1, the tree may only contain unique keys. */ -static inline struct eb32_node * +static forceinline struct eb32_node * __eb32i_insert(struct eb_root *root, struct eb32_node *new) { struct eb32_node *old; unsigned int side; diff --git a/include/common/eb64tree.h b/include/common/eb64tree.h index 767ba834b..3874ac8d9 100644 --- a/include/common/eb64tree.h +++ b/include/common/eb64tree.h @@ -109,7 +109,7 @@ REGPRM2 struct eb64_node *eb64i_insert(struct eb_root *root, struct eb64_node *n */ /* Delete node from the tree if it was linked in. Mark the node unused. */ -static inline void __eb64_delete(struct eb64_node *eb64) +static forceinline void __eb64_delete(struct eb64_node *eb64) { __eb_delete(&eb64->node); } @@ -118,7 +118,7 @@ static inline void __eb64_delete(struct eb64_node *eb64) * Find the first occurence of a key in the tree . If none can be * found, return NULL. */ -static inline struct eb64_node *__eb64_lookup(struct eb_root *root, u64 x) +static forceinline struct eb64_node *__eb64_lookup(struct eb_root *root, u64 x) { struct eb64_node *node; eb_troot_t *troot; @@ -162,7 +162,7 @@ static inline struct eb64_node *__eb64_lookup(struct eb_root *root, u64 x) * Find the first occurence of a signed key in the tree . If none can * be found, return NULL. */ -static inline struct eb64_node *__eb64i_lookup(struct eb_root *root, s64 x) +static forceinline struct eb64_node *__eb64i_lookup(struct eb_root *root, s64 x) { struct eb64_node *node; eb_troot_t *troot; @@ -207,7 +207,7 @@ static inline struct eb64_node *__eb64i_lookup(struct eb_root *root, s64 x) * Only new->key needs be set with the key. The eb64_node is returned. * If root->b[EB_RGHT]==1, the tree may only contain unique keys. */ -static inline struct eb64_node * +static forceinline struct eb64_node * __eb64_insert(struct eb_root *root, struct eb64_node *new) { struct eb64_node *old; unsigned int side; @@ -381,7 +381,7 @@ __eb64_insert(struct eb_root *root, struct eb64_node *new) { * signed keys. Only new->key needs be set with the key. The eb64_node * is returned. If root->b[EB_RGHT]==1, the tree may only contain unique keys. */ -static inline struct eb64_node * +static forceinline struct eb64_node * __eb64i_insert(struct eb_root *root, struct eb64_node *new) { struct eb64_node *old; unsigned int side; diff --git a/include/common/ebpttree.h b/include/common/ebpttree.h index a5ea0bd35..d863f3ce8 100644 --- a/include/common/ebpttree.h +++ b/include/common/ebpttree.h @@ -110,7 +110,7 @@ REGPRM2 struct ebpt_node *ebpt_insert(struct eb_root *root, struct ebpt_node *ne */ /* Delete node from the tree if it was linked in. Mark the node unused. */ -static inline void __ebpt_delete(struct ebpt_node *ebpt) +static forceinline void __ebpt_delete(struct ebpt_node *ebpt) { __eb_delete(&ebpt->node); } @@ -119,7 +119,7 @@ static inline void __ebpt_delete(struct ebpt_node *ebpt) * Find the first occurence of a key in the tree . If none can be * found, return NULL. */ -static inline struct ebpt_node *__ebpt_lookup(struct eb_root *root, void *x) +static forceinline struct ebpt_node *__ebpt_lookup(struct eb_root *root, void *x) { struct ebpt_node *node; eb_troot_t *troot; @@ -163,7 +163,7 @@ static inline struct ebpt_node *__ebpt_lookup(struct eb_root *root, void *x) * Only new->key needs be set with the key. The ebpt_node is returned. * If root->b[EB_RGHT]==1, the tree may only contain unique keys. */ -static inline struct ebpt_node * +static forceinline struct ebpt_node * __ebpt_insert(struct eb_root *root, struct ebpt_node *new) { struct ebpt_node *old; unsigned int side; diff --git a/include/common/ebtree.h b/include/common/ebtree.h index b737bc915..a2024bc9d 100644 --- a/include/common/ebtree.h +++ b/include/common/ebtree.h @@ -333,6 +333,17 @@ static inline int fls64(unsigned long long x) #endif #endif +/* By default, gcc does not inline large chunks of code, but we want it to + * respect our choices. + */ +#if !defined(forceinline) +#if __GNUC__ < 3 +#define forceinline inline +#else +#define forceinline inline __attribute__((always_inline)) +#endif +#endif + /* Support passing function parameters in registers. For this, the * CONFIG_EBTREE_REGPARM macro has to be set to the maximal number of registers * allowed. Some functions have intentionally received a regparm lower than @@ -492,7 +503,7 @@ static inline struct eb_node *eb_walk_down(eb_troot_t *start, unsigned int side) * a subtree of at least 2 entries. It will probably never be needed inlined, * and it is not for end-user. */ -static inline struct eb_node * +static forceinline struct eb_node * __eb_insert_dup(struct eb_node *sub, struct eb_node *new) { struct eb_node *head = sub; @@ -658,7 +669,7 @@ static inline struct eb_node *eb_next_unique(struct eb_node *node) /* Removes a leaf node from the tree if it was still in it. Marks the node * as unlinked. */ -static inline void __eb_delete(struct eb_node *node) +static forceinline void __eb_delete(struct eb_node *node) { __label__ delete_unlink; unsigned int pside, gpside, sibtype;