From c4262961f8c47b67b22643b9fa4c854246ba7f3e Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Mon, 10 May 2010 23:42:40 +0200 Subject: [PATCH] [MEDIUM] acl: add tree-based lookups of exact strings Now if some ACL patterns are loaded from a file and the operation is an exact string match, the data will be arranged in a tree, yielding a significant performance boost on large data sets. Note that this only works when case is sensitive. A new dedicated function, acl_lookup_str(), has been created for this matching. It is called for every possible input data to test and it looks the tree up for the data. Since the keywords are loosely typed, we would have had to add a new columns to all keywords to adjust the function depending on the type. Instead, we just compare on the match function. We call acl_lookup_str() when we could use acl_match_str(). The tree lookup is performed first, then the remaining patterns are attempted if the tree returned nothing. A quick test shows that when matching a header against a list of 52000 network names, haproxy uses 68% of one core on a core2-duo 3.2 GHz at 42000 requests per second, versus 66% without any rule, which means only a 2% CPU increase for 52000 rules. Doing the same test without the tree leads to 100% CPU at 6900 requests/s. Also it was possible to run the same test at full speed with about 50 sets of 52000 rules without any measurable performance drop. --- src/acl.c | 44 ++++++++++++++++++++++++++++++++++++++++++++ src/proto_http.c | 12 ++++++------ src/proto_tcp.c | 2 +- 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/src/acl.c b/src/acl.c index ad3feaf91..b474c82c3 100644 --- a/src/acl.c +++ b/src/acl.c @@ -25,6 +25,8 @@ #include #include +#include + /* The capabilities of filtering hooks describe the type of information * available to each of them. */ @@ -130,6 +132,25 @@ int acl_match_str(struct acl_test *test, struct acl_pattern *pattern) return ACL_PAT_FAIL; } +/* Lookup a string in the expression's pattern tree. The node is returned if it + * exists, otherwise NULL. + */ +void *acl_lookup_str(struct acl_test *test, struct acl_expr *expr) +{ + /* data are stored in a tree */ + struct ebmb_node *node; + char prev; + + /* we may have to force a trailing zero on the test pattern */ + prev = test->ptr[test->len]; + if (prev) + test->ptr[test->len] = '\0'; + node = ebst_lookup(&expr->pattern_tree, test->ptr); + if (prev) + test->ptr[test->len] = prev; + return node; +} + /* Executes a regex. It needs to change the data. If it is marked READ_ONLY * then it will be allocated and duplicated in place so that others may use * it later on. Note that this is embarrassing because we always try to avoid @@ -329,6 +350,23 @@ int acl_parse_str(const char **text, struct acl_pattern *pattern, int *opaque) int len; len = strlen(*text); + + if (pattern->flags & ACL_PAT_F_TREE_OK) { + /* we're allowed to put the data in a tree whose root is pointed + * to by val.tree. + */ + struct ebmb_node *node; + + node = calloc(1, sizeof(*node) + len + 1); + if (!node) + return 0; + memcpy(node->key, *text, len + 1); + if (ebst_insert(pattern->val.tree, node) != node) + free(node); /* was a duplicate */ + pattern->flags |= ACL_PAT_F_TREE; /* this pattern now contains a tree */ + return 1; + } + pattern->ptr.str = strdup(*text); if (!pattern->ptr.str) return 0; @@ -1224,6 +1262,12 @@ int acl_exec_cond(struct acl_cond *cond, struct proxy *px, struct session *l4, v acl_res |= ACL_PAT_FAIL; } else { + if (expr->pattern_tree.b[EB_LEFT]) { + /* a tree is present, let's check what type it is */ + if (expr->kw->match == acl_match_str) + acl_res |= acl_lookup_str(&test, expr) ? ACL_PAT_PASS : ACL_PAT_FAIL; + } + /* call the match() function for all tests on this value */ list_for_each_entry(pattern, &expr->patterns, list) { if (acl_res == ACL_PAT_PASS) diff --git a/src/proto_http.c b/src/proto_http.c index 882f0dc4d..c8835a5b5 100644 --- a/src/proto_http.c +++ b/src/proto_http.c @@ -7360,11 +7360,11 @@ static struct acl_kw_list acl_kws = {{ },{ { "req_proto_http", acl_parse_nothing, acl_fetch_proto_http, acl_match_nothing, ACL_USE_L7REQ_PERMANENT }, { "method", acl_parse_meth, acl_fetch_meth, acl_match_meth, ACL_USE_L7REQ_PERMANENT }, - { "req_ver", acl_parse_ver, acl_fetch_rqver, acl_match_str, ACL_USE_L7REQ_VOLATILE }, - { "resp_ver", acl_parse_ver, acl_fetch_stver, acl_match_str, ACL_USE_L7RTR_VOLATILE }, + { "req_ver", acl_parse_ver, acl_fetch_rqver, acl_match_str, ACL_USE_L7REQ_VOLATILE|ACL_MAY_LOOKUP }, + { "resp_ver", acl_parse_ver, acl_fetch_stver, acl_match_str, ACL_USE_L7RTR_VOLATILE|ACL_MAY_LOOKUP }, { "status", acl_parse_int, acl_fetch_stcode, acl_match_int, ACL_USE_L7RTR_PERMANENT }, - { "url", acl_parse_str, acl_fetch_url, acl_match_str, ACL_USE_L7REQ_VOLATILE }, + { "url", acl_parse_str, acl_fetch_url, acl_match_str, ACL_USE_L7REQ_VOLATILE|ACL_MAY_LOOKUP }, { "url_beg", acl_parse_str, acl_fetch_url, acl_match_beg, ACL_USE_L7REQ_VOLATILE }, { "url_end", acl_parse_str, acl_fetch_url, acl_match_end, ACL_USE_L7REQ_VOLATILE }, { "url_sub", acl_parse_str, acl_fetch_url, acl_match_sub, ACL_USE_L7REQ_VOLATILE }, @@ -7375,7 +7375,7 @@ static struct acl_kw_list acl_kws = {{ },{ { "url_port", acl_parse_int, acl_fetch_url_port, acl_match_int, ACL_USE_L7REQ_VOLATILE }, /* note: we should set hdr* to use ACL_USE_HDR_VOLATILE, and chdr* to use L7REQ_VOLATILE */ - { "hdr", acl_parse_str, acl_fetch_chdr, acl_match_str, ACL_USE_L7REQ_VOLATILE }, + { "hdr", acl_parse_str, acl_fetch_chdr, acl_match_str, ACL_USE_L7REQ_VOLATILE|ACL_MAY_LOOKUP }, { "hdr_reg", acl_parse_reg, acl_fetch_chdr, acl_match_reg, ACL_USE_L7REQ_VOLATILE }, { "hdr_beg", acl_parse_str, acl_fetch_chdr, acl_match_beg, ACL_USE_L7REQ_VOLATILE }, { "hdr_end", acl_parse_str, acl_fetch_chdr, acl_match_end, ACL_USE_L7REQ_VOLATILE }, @@ -7386,7 +7386,7 @@ static struct acl_kw_list acl_kws = {{ },{ { "hdr_val", acl_parse_int, acl_fetch_chdr_val,acl_match_int, ACL_USE_L7REQ_VOLATILE }, { "hdr_ip", acl_parse_ip, acl_fetch_chdr_ip, acl_match_ip, ACL_USE_L7REQ_VOLATILE }, - { "shdr", acl_parse_str, acl_fetch_shdr, acl_match_str, ACL_USE_L7RTR_VOLATILE }, + { "shdr", acl_parse_str, acl_fetch_shdr, acl_match_str, ACL_USE_L7RTR_VOLATILE|ACL_MAY_LOOKUP }, { "shdr_reg", acl_parse_reg, acl_fetch_shdr, acl_match_reg, ACL_USE_L7RTR_VOLATILE }, { "shdr_beg", acl_parse_str, acl_fetch_shdr, acl_match_beg, ACL_USE_L7RTR_VOLATILE }, { "shdr_end", acl_parse_str, acl_fetch_shdr, acl_match_end, ACL_USE_L7RTR_VOLATILE }, @@ -7397,7 +7397,7 @@ static struct acl_kw_list acl_kws = {{ },{ { "shdr_val", acl_parse_int, acl_fetch_shdr_val,acl_match_int, ACL_USE_L7RTR_VOLATILE }, { "shdr_ip", acl_parse_ip, acl_fetch_shdr_ip, acl_match_ip, ACL_USE_L7RTR_VOLATILE }, - { "path", acl_parse_str, acl_fetch_path, acl_match_str, ACL_USE_L7REQ_VOLATILE }, + { "path", acl_parse_str, acl_fetch_path, acl_match_str, ACL_USE_L7REQ_VOLATILE|ACL_MAY_LOOKUP }, { "path_reg", acl_parse_reg, acl_fetch_path, acl_match_reg, ACL_USE_L7REQ_VOLATILE }, { "path_beg", acl_parse_str, acl_fetch_path, acl_match_beg, ACL_USE_L7REQ_VOLATILE }, { "path_end", acl_parse_str, acl_fetch_path, acl_match_end, ACL_USE_L7REQ_VOLATILE }, diff --git a/src/proto_tcp.c b/src/proto_tcp.c index 6494db7a5..7c958fc4d 100644 --- a/src/proto_tcp.c +++ b/src/proto_tcp.c @@ -1138,7 +1138,7 @@ static struct cfg_kw_list cfg_kws = {{ },{ static struct acl_kw_list acl_kws = {{ },{ { "req_len", acl_parse_int, acl_fetch_req_len, acl_match_int, ACL_USE_L4REQ_VOLATILE }, { "req_ssl_ver", acl_parse_dotted_ver, acl_fetch_req_ssl_ver, acl_match_int, ACL_USE_L4REQ_VOLATILE }, - { "req_rdp_cookie", acl_parse_str, acl_fetch_rdp_cookie, acl_match_str, ACL_USE_L4REQ_VOLATILE }, + { "req_rdp_cookie", acl_parse_str, acl_fetch_rdp_cookie, acl_match_str, ACL_USE_L4REQ_VOLATILE|ACL_MAY_LOOKUP }, { "req_rdp_cookie_cnt", acl_parse_int, acl_fetch_rdp_cookie_cnt, acl_match_int, ACL_USE_L4REQ_VOLATILE }, { NULL, NULL, NULL, NULL }, }};