MINOR: http: implement the max-keep-alive-queue setting

Finn Arne Gangstad suggested that we should have the ability to break
keep-alive when the target server has reached its maxconn and that a
number of connections are present in the queue. After some discussion
around his proposed patch, the following solution was suggested : have
a per-proxy setting to fix a limit to the number of queued connections
on a server after which we break keep-alive. This ensures that even in
high latency networks where keep-alive is beneficial, we try to find a
different server.

This patch is partially based on his original proposal and implements
this configurable threshold.

doc/configuration.txt

@@ -1275,6 +1275,7 @@ http-send-name-header                     -          -         X         X
 id                                        -          X         X         X
 ignore-persist                            -          X         X         X
 log                                  (*)  X          X         X         X
+max-keep-alive-queue                      X          -         X         X
 maxconn                                   X          X         X         -
 mode                                      X          X         X         X
 monitor fail                              -          X         X         -
@@ -3379,6 +3380,34 @@ log-format <string>
    See also : Custom Log Format (8.2.4)
 
 
+max-keep-alive-queue <value>
+  Set the maximum server queue size for maintaining keep-alive connections
+  May be used in sections:    defaults | frontend | listen | backend
+                                 yes   |     no   |   yes  |   yes
+
+  HTTP keep-alive tries to reuse the same server connection whenever possible,
+  but sometimes it can be counter-productive, for example if a server has a lot
+  of connections while other ones are idle. This is especially true for static
+  servers.
+
+  The purpose of this setting is to set a threshold on the number of queued
+  connections at which haproxy stops trying to reuse the same server and prefers
+  to find another one. The default value, -1, means there is no limit. A value
+  of zero means that keep-alive requests will never be queued. For very close
+  servers which can be reached with a low latency and which are not sensible to
+  breaking keep-alive, a low value is recommended (eg: local static server can
+  use a value of 10 or less). For remote servers suffering from a high latency,
+  higher values might be needed to cover for the latency and/or the cost of
+  picking a different server.
+
+  Note that this has no impact on responses which are maintained to the same
+  server consecutively to a 401 response. They will still go to the same server
+  even if they have to be queued.
+
+  See also : "option http-server-close", "option prefer-last-server", server
+             "maxconn" and cookie persistence.
+
+
 maxconn <conns>
   Fix the maximum number of concurrent connections on a frontend
   May be used in sections :   defaults | frontend | listen | backend

include/proto/queue.h

@@ -65,6 +65,11 @@ static inline struct pendconn *pendconn_from_px(const struct proxy *px) {
 	return LIST_ELEM(px->pendconns.n, struct pendconn *, list);
 }
 
+/* Returns 0 if all slots are full on a server, or 1 if there are slots available. */
+static inline int server_has_room(const struct server *s) {
+	return !s->maxconn || s->cur_sess < srv_dynamic_maxconn(s);
+}
+
 /* returns 0 if nothing has to be done for server <s> regarding queued connections,
  * and non-zero otherwise. If the server is down, we only check its own queue. Suited
  * for and if/else usage.

include/types/proxy.h

@@ -289,6 +289,7 @@ struct proxy {
 	struct list pendconns;			/* pending connections with no server assigned yet */
 	int nbpend;				/* number of pending connections with no server assigned yet */
 	int totpend;				/* total number of pending connections on this instance (for stats) */
+	int max_ka_queue;			/* 1+maximum requests in queue accepted for reusing a K-A conn (0=none) */
 	unsigned int feconn, beconn;		/* # of active frontend and backends sessions */
 	struct freq_ctr fe_req_per_sec;		/* HTTP requests per second on the frontend */
 	struct freq_ctr fe_conn_per_sec;	/* received connections per second on the frontend */

src/backend.c

@@ -541,8 +541,12 @@ int assign_server(struct session *s)
 
 	if (conn &&
 	    (conn->flags & CO_FL_CONNECTED) &&
-	    ((s->be->options & PR_O_PREF_LAST) || (s->txn.flags & TX_PREFER_LAST)) &&
 	    objt_server(conn->target) && __objt_server(conn->target)->proxy == s->be &&
+	    ((s->txn.flags & TX_PREFER_LAST) ||
+	     ((s->be->options & PR_O_PREF_LAST) &&
+	      (!s->be->max_ka_queue ||
+	       server_has_room(__objt_server(conn->target)) ||
+	       (__objt_server(conn->target)->nbpend + 1) < s->be->max_ka_queue))) &&
 	    srv_is_usable(__objt_server(conn->target)->state, __objt_server(conn->target)->eweight)) {
 		/* This session was relying on a server in a previous request
 		 * and the proxy has "option prefer-current-server" set, so

src/cfgparse.c

@@ -1930,6 +1930,7 @@ int cfg_parse_listen(const char *file, int linenum, char **args, int kwm)
 		if (curproxy->cap & PR_CAP_BE) {
 			curproxy->fullconn = defproxy.fullconn;
 			curproxy->conn_retries = defproxy.conn_retries;
+			curproxy->max_ka_queue = defproxy.max_ka_queue;
 
 			if (defproxy.check_req) {
 				curproxy->check_req = calloc(1, defproxy.check_len);

src/proxy.c

@@ -275,6 +275,45 @@ static int proxy_parse_rate_limit(char **args, int section, struct proxy *proxy,
 	return retval;
 }
 
+/* This function parses a "max-keep-alive-queue" statement in a proxy section.
+ * It returns -1 if there is any error, 1 for a warning, otherwise zero. If it
+ * does not return zero, it will write an error or warning message into a
+ * preallocated buffer returned at <err>. The function must be called with
+ * <args> pointing to the first command line word, with <proxy> pointing to
+ * the proxy being parsed, and <defpx> to the default proxy or NULL.
+ */
+static int proxy_parse_max_ka_queue(char **args, int section, struct proxy *proxy,
+                                    struct proxy *defpx, const char *file, int line,
+                                    char **err)
+{
+	int retval;
+	char *res;
+	unsigned int val;
+
+	retval = 0;
+
+	if (*args[1] == 0) {
+		memprintf(err, "'%s' expects expects an integer value (or -1 to disable)", args[0]);
+		return -1;
+	}
+
+	val = strtol(args[1], &res, 0);
+	if (*res) {
+		memprintf(err, "'%s' : unexpected character '%c' in integer value '%s'", args[0], *res, args[1]);
+		return -1;
+	}
+
+	if (!(proxy->cap & PR_CAP_BE)) {
+		memprintf(err, "%s will be ignored because %s '%s' has no backend capability",
+		          args[0], proxy_type_str(proxy), proxy->id);
+		retval = 1;
+	}
+
+	/* we store <val+1> so that a user-facing value of -1 is stored as zero (default) */
+	proxy->max_ka_queue = val + 1;
+	return retval;
+}
+
 /* This function inserts proxy <px> into the tree of known proxies. The proxy's
  * name is used as the storing key so it must already have been initialized.
  */
@@ -926,6 +965,7 @@ static struct cfg_kw_list cfg_kws = {ILH, {
 	{ CFG_LISTEN, "contimeout", proxy_parse_timeout },
 	{ CFG_LISTEN, "srvtimeout", proxy_parse_timeout },
 	{ CFG_LISTEN, "rate-limit", proxy_parse_rate_limit },
+	{ CFG_LISTEN, "max-keep-alive-queue", proxy_parse_max_ka_queue },
 	{ 0, NULL, NULL },
 }};