From 33cb0653480f888e30e9dccd189a7f8b3fd64eb2 Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Tue, 23 Dec 2014 22:52:37 +0100 Subject: [PATCH] MINOR: config: implement global setting tune.buffers.limit This setting is used to limit memory usage without causing the alloc failures caused by "-m". Unexpectedly, tests have shown a performance boost of up to about 18% on HTTP traffic when limiting the number of buffers to about 10% of the amount of concurrent connections. tune.buffers.limit Sets a hard limit on the number of buffers which may be allocated per process. The default value is zero which means unlimited. The minimum non-zero value will always be greater than "tune.buffers.reserve" and should ideally always be about twice as large. Forcing this value can be particularly useful to limit the amount of memory a process may take, while retaining a sane behaviour. When this limit is reached, sessions which need a buffer wait for another one to be released by another session. Since buffers are dynamically allocated and released, the waiting time is very short and not perceptible provided that limits remain reasonable. In fact sometimes reducing the limit may even increase performance by increasing the CPU cache's efficiency. Tests have shown good results on average HTTP traffic with a limit to 1/10 of the expected global maxconn setting, which also significantly reduces memory usage. The memory savings come from the fact that a number of connections will not allocate 2*tune.bufsize. It is best not to touch this value unless advised to do so by an haproxy core developer. --- doc/configuration.txt | 17 +++++++++++++++++ include/types/global.h | 1 + src/buffer.c | 2 ++ src/cfgparse.c | 16 ++++++++++++++++ 4 files changed, 36 insertions(+) diff --git a/doc/configuration.txt b/doc/configuration.txt index d3f049eab..d8063babe 100644 --- a/doc/configuration.txt +++ b/doc/configuration.txt @@ -891,6 +891,23 @@ spread-checks <0..50, in percent> and +/- 50%. A value between 2 and 5 seems to show good results. The default value remains at 0. +tune.buffers.limit + Sets a hard limit on the number of buffers which may be allocated per process. + The default value is zero which means unlimited. The minimum non-zero value + will always be greater than "tune.buffers.reserve" and should ideally always + be about twice as large. Forcing this value can be particularly useful to + limit the amount of memory a process may take, while retaining a sane + behaviour. When this limit is reached, sessions which need a buffer wait for + another one to be released by another session. Since buffers are dynamically + allocated and released, the waiting time is very short and not perceptible + provided that limits remain reasonable. In fact sometimes reducing the limit + may even increase performance by increasing the CPU cache's efficiency. Tests + have shown good results on average HTTP traffic with a limit to 1/10 of the + expected global maxconn setting, which also significantly reduces memory + usage. The memory savings come from the fact that a number of connections + will not allocate 2*tune.bufsize. It is best not to touch this value unless + advised to do so by an haproxy core developer. + tune.buffers.reserve Sets the number of buffers which are pre-allocated and reserved for use only during memory shortage conditions resulting in failed memory allocations. The diff --git a/include/types/global.h b/include/types/global.h index 143b8647a..22490b06b 100644 --- a/include/types/global.h +++ b/include/types/global.h @@ -129,6 +129,7 @@ struct global { int bufsize; /* buffer size in bytes, defaults to BUFSIZE */ int maxrewrite; /* buffer max rewrite size in bytes, defaults to MAXREWRITE */ int reserved_bufs; /* how many buffers can only be allocated for response */ + int buf_limit; /* if not null, how many total buffers may only be allocated */ int client_sndbuf; /* set client sndbuf to this value if not null */ int client_rcvbuf; /* set client rcvbuf to this value if not null */ int server_sndbuf; /* set server sndbuf to this value if not null */ diff --git a/src/buffer.c b/src/buffer.c index c7326dcd6..e156991e6 100644 --- a/src/buffer.c +++ b/src/buffer.c @@ -48,6 +48,8 @@ int init_buffer() * release a server connection). */ pool2_buffer->minavail = MAX(global.tune.reserved_bufs, 3); + if (global.tune.buf_limit) + pool2_buffer->limit = global.tune.buf_limit; buffer = pool_refill_alloc(pool2_buffer, pool2_buffer->minavail - 1); if (!buffer) diff --git a/src/cfgparse.c b/src/cfgparse.c index ad0b44bf8..99ac8a426 100644 --- a/src/cfgparse.c +++ b/src/cfgparse.c @@ -696,6 +696,20 @@ int cfg_parse_global(const char *file, int linenum, char **args, int kwm) } } #endif + else if (!strcmp(args[0], "tune.buffers.limit")) { + if (*(args[1]) == 0) { + Alert("parsing [%s:%d] : '%s' expects an integer argument.\n", file, linenum, args[0]); + err_code |= ERR_ALERT | ERR_FATAL; + goto out; + } + global.tune.buf_limit = atol(args[1]); + if (global.tune.buf_limit) { + if (global.tune.buf_limit < 3) + global.tune.buf_limit = 3; + if (global.tune.buf_limit <= global.tune.reserved_bufs) + global.tune.buf_limit = global.tune.reserved_bufs + 1; + } + } else if (!strcmp(args[0], "tune.buffers.reserve")) { if (*(args[1]) == 0) { Alert("parsing [%s:%d] : '%s' expects an integer argument.\n", file, linenum, args[0]); @@ -705,6 +719,8 @@ int cfg_parse_global(const char *file, int linenum, char **args, int kwm) global.tune.reserved_bufs = atol(args[1]); if (global.tune.reserved_bufs < 2) global.tune.reserved_bufs = 2; + if (global.tune.buf_limit && global.tune.buf_limit <= global.tune.reserved_bufs) + global.tune.buf_limit = global.tune.reserved_bufs + 1; } else if (!strcmp(args[0], "tune.bufsize")) { if (*(args[1]) == 0) {