From e803de2c6baf07b95a4432f1c99af8ac761f58be Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Thu, 21 Jan 2010 17:43:04 +0100 Subject: [PATCH] [MINOR] add the ability to force kernel socket buffer size. Sometimes we need to be able to change the default kernel socket buffer size (recv and send). Four new global settings have been added for this : - tune.rcvbuf.client - tune.rcvbuf.server - tune.sndbuf.client - tune.sndbuf.server Those can be used to reduce kernel memory footprint with large numbers of concurrent connections, and to reduce risks of write timeouts with very slow clients due to excessive kernel buffering. --- doc/configuration.txt | 27 ++++++++++++++++++++++ include/types/global.h | 4 ++++ src/cfgparse.c | 52 ++++++++++++++++++++++++++++++++++++++++++ src/client.c | 6 +++++ src/proto_tcp.c | 6 +++++ 5 files changed, 95 insertions(+) diff --git a/doc/configuration.txt b/doc/configuration.txt index 9d9807485..d626ade86 100644 --- a/doc/configuration.txt +++ b/doc/configuration.txt @@ -399,6 +399,10 @@ The following keywords are supported in the "global" section : - tune.maxaccept - tune.maxpollevents - tune.maxrewrite + - tune.rcvbuf.client + - tune.rcvbuf.server + - tune.sndbuf.client + - tune.sndbuf.server * Debugging - debug @@ -639,6 +643,29 @@ tune.maxrewrite larger than that. This means you don't have to worry about it when changing bufsize. +tune.rcvbuf.client +tune.rcvbuf.server + Forces the kernel socket receive buffer size on the client or the server side + to the specified value in bytes. This value applies to all TCP/HTTP frontends + and backends. It should normally never be set, and the default size (0) lets + the kernel autotune this value depending on the amount of available memory. + However it can sometimes help to set it to very low values (eg: 4096) in + order to save kernel memory by preventing it from buffering too large amounts + of received data. Lower values will significantly increase CPU usage though. + +tune.sndbuf.client +tune.sndbuf.server + Forces the kernel socket send buffer size on the client or the server side to + the specified value in bytes. This value applies to all TCP/HTTP frontends + and backends. It should normally never be set, and the default size (0) lets + the kernel autotune this value depending on the amount of available memory. + However it can sometimes help to set it to very low values (eg: 4096) in + order to save kernel memory by preventing it from buffering too large amounts + of received data. Lower values will significantly increase CPU usage though. + Another use case is to prevent write timeouts with extremely slow clients due + to the kernel waiting for a large part of the buffer to be read before + notifying haproxy again. + 3.3. Debugging -------------- diff --git a/include/types/global.h b/include/types/global.h index 2a7bc4609..463bd3876 100644 --- a/include/types/global.h +++ b/include/types/global.h @@ -87,6 +87,10 @@ struct global { int recv_enough; /* how many input bytes at once are "enough" */ int bufsize; /* buffer size in bytes, defaults to BUFSIZE */ int maxrewrite; /* buffer max rewrite size in bytes, defaults to MAXREWRITE */ + int client_sndbuf; /* set client sndbuf to this value if not null */ + int client_rcvbuf; /* set client rcvbuf to this value if not null */ + int server_sndbuf; /* set server sndbuf to this value if not null */ + int server_rcvbuf; /* set server rcvbuf to this value if not null */ } tune; struct listener stats_sock; /* unix socket listener for statistics */ struct proxy *stats_fe; /* the frontend holding the stats settings */ diff --git a/src/cfgparse.c b/src/cfgparse.c index 39a79f2f7..9aae768ab 100644 --- a/src/cfgparse.c +++ b/src/cfgparse.c @@ -475,6 +475,58 @@ int cfg_parse_global(const char *file, int linenum, char **args, int kwm) if (global.tune.maxrewrite >= global.tune.bufsize / 2) global.tune.maxrewrite = global.tune.bufsize / 2; } + else if (!strcmp(args[0], "tune.rcvbuf.client")) { + if (global.tune.client_rcvbuf != 0) { + Alert("parsing [%s:%d] : '%s' already specified. Continuing.\n", file, linenum, args[0]); + err_code |= ERR_ALERT; + goto out; + } + if (*(args[1]) == 0) { + Alert("parsing [%s:%d] : '%s' expects an integer argument.\n", file, linenum, args[0]); + err_code |= ERR_ALERT | ERR_FATAL; + goto out; + } + global.tune.client_rcvbuf = atol(args[1]); + } + else if (!strcmp(args[0], "tune.rcvbuf.server")) { + if (global.tune.server_rcvbuf != 0) { + Alert("parsing [%s:%d] : '%s' already specified. Continuing.\n", file, linenum, args[0]); + err_code |= ERR_ALERT; + goto out; + } + if (*(args[1]) == 0) { + Alert("parsing [%s:%d] : '%s' expects an integer argument.\n", file, linenum, args[0]); + err_code |= ERR_ALERT | ERR_FATAL; + goto out; + } + global.tune.server_rcvbuf = atol(args[1]); + } + else if (!strcmp(args[0], "tune.sndbuf.client")) { + if (global.tune.client_sndbuf != 0) { + Alert("parsing [%s:%d] : '%s' already specified. Continuing.\n", file, linenum, args[0]); + err_code |= ERR_ALERT; + goto out; + } + if (*(args[1]) == 0) { + Alert("parsing [%s:%d] : '%s' expects an integer argument.\n", file, linenum, args[0]); + err_code |= ERR_ALERT | ERR_FATAL; + goto out; + } + global.tune.client_sndbuf = atol(args[1]); + } + else if (!strcmp(args[0], "tune.sndbuf.server")) { + if (global.tune.server_sndbuf != 0) { + Alert("parsing [%s:%d] : '%s' already specified. Continuing.\n", file, linenum, args[0]); + err_code |= ERR_ALERT; + goto out; + } + if (*(args[1]) == 0) { + Alert("parsing [%s:%d] : '%s' expects an integer argument.\n", file, linenum, args[0]); + err_code |= ERR_ALERT | ERR_FATAL; + goto out; + } + global.tune.server_sndbuf = atol(args[1]); + } else if (!strcmp(args[0], "uid")) { if (global.uid != 0) { Alert("parsing [%s:%d] : user/uid already specified. Continuing.\n", file, linenum); diff --git a/src/client.c b/src/client.c index 458bb1c2c..f27424534 100644 --- a/src/client.c +++ b/src/client.c @@ -172,6 +172,12 @@ int event_accept(int fd) { if (p->options & PR_O_TCP_NOLING) setsockopt(cfd, SOL_SOCKET, SO_LINGER, (struct linger *) &nolinger, sizeof(struct linger)); + if (global.tune.client_sndbuf) + setsockopt(cfd, SOL_SOCKET, SO_SNDBUF, &global.tune.client_sndbuf, sizeof(global.tune.client_sndbuf)); + + if (global.tune.client_rcvbuf) + setsockopt(cfd, SOL_SOCKET, SO_RCVBUF, &global.tune.client_rcvbuf, sizeof(global.tune.client_rcvbuf)); + t->process = l->handler; t->context = s; t->nice = l->nice; diff --git a/src/proto_tcp.c b/src/proto_tcp.c index b2efa0b78..23f3ac93c 100644 --- a/src/proto_tcp.c +++ b/src/proto_tcp.c @@ -378,6 +378,12 @@ int tcpv4_connect_server(struct stream_interface *si, setsockopt(fd, IPPROTO_TCP, TCP_QUICKACK, (char *) &zero, sizeof(zero)); #endif + if (global.tune.server_sndbuf) + setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &global.tune.server_sndbuf, sizeof(global.tune.server_sndbuf)); + + if (global.tune.server_rcvbuf) + setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &global.tune.server_rcvbuf, sizeof(global.tune.server_rcvbuf)); + if ((connect(fd, (struct sockaddr *)srv_addr, sizeof(struct sockaddr_in)) == -1) && (errno != EINPROGRESS) && (errno != EALREADY) && (errno != EISCONN)) {