MEDIUM: server/mux-h2: implement idle-ping on backend side

This commit implements support for idle-ping on the backend side. First,
a new server keyword "idle-ping" is defined in configuration parsing. It
is used to set the corresponding new server member.

The second part of this commit implements idle-ping support on H2 MUX. A
new inlined function conn_idle_ping() is defined to access connection
idle-ping value. Two new connection flags are defined H2_CF_IDL_PING and
H2_CF_IDL_PING_SENT. The first one is set for idle connections via
h2c_update_timeout().

On h2_timeout_task() handler, if first flag is set, instead of releasing
the connection as before, the second flag is set and tasklet is
scheduled. As both flags are now set, h2_process_mux() will proceed to
PING emission. The timer has also been rearmed to the idle-ping value.
If a PING ACK is received before next timeout, connection timer is
refreshed. Else, the connection is released, as with timer expiration.

Also of importance, special care is needed when a backend connection is
going to idle. In this case, idle-ping timer must be rearmed. Thus a new
invokation of h2c_update_timeout() is performed on h2_detach().

doc/configuration.txt

@@ -18639,6 +18639,18 @@ id <value>
   the proxy. An unused ID will automatically be assigned if unset. The first
   assigned value will be 1. This ID is currently only returned in statistics.
 
+idle-ping <delay>
+  May be used in the following contexts: tcp, http, log
+
+  Define an interval for periodic liveliness on idle backend connections. If
+  the peer is unable to respond before the next scheduled test, the connection
+  is closed. This keyword refers to the backend side, so it is useful to check
+  that idle connections are still usable. Note that this won't prevent the
+  connection from being destroyed on idle pool purge.
+
+  This feature relies on specific underlying protocol support. For now, only H2
+  mux implements it. Idle-ping is simply ignored by other protocols.
+
 init-addr {last | libc | none | <ip>},[...]*
   May be used in the following contexts: tcp, http, log
 

include/haproxy/connection.h

@@ -708,6 +708,19 @@ static inline void conn_set_reverse(struct connection *conn, enum obj_type *targ
 	conn->reverse.target = target;
 }
 
+/* Returns idle-ping value for <conn> depending on its proxy side. */
+static inline int conn_idle_ping(const struct connection *conn)
+{
+	if (conn_is_back(conn)) {
+		struct server *srv = objt_server(conn->target);
+		return srv ? srv->idle_ping : TICK_ETERNITY;
+	}
+	else {
+		/* TODO */
+		return TICK_ETERNITY;
+	}
+}
+
 /* Returns the listener instance for connection used for active reverse. */
 static inline struct listener *conn_active_reverse_listener(const struct connection *conn)
 {

include/haproxy/mux_h2-t.h

@@ -70,6 +70,9 @@
 #define H2_CF_ERROR             0x01000000  //A read error was detected (handled has an abort)
 #define H2_CF_WAIT_INLIST       0x02000000  // there is at least one stream blocked by another stream in send_list/fctl_list
 
+#define H2_CF_IDL_PING          0x04000000  // timer task scheduled for a PING emission
+#define H2_CF_IDL_PING_SENT     0x08000000  // PING emitted, or will be on next tasklet run, waiting for ACK
+
 /* This function is used to report flags in debugging tools. Please reflect
  * below any single-bit flag addition above in the same order via the
  * __APPEND_FLAG macro. The new end of the buffer is returned.

include/haproxy/server-t.h

@@ -340,6 +340,7 @@ struct server {
 	short onmarkeddown;			/* what to do when marked down: one of HANA_ONMARKEDDOWN_* */
 	short onmarkedup;			/* what to do when marked up: one of HANA_ONMARKEDUP_* */
 	int slowstart;				/* slowstart time in seconds (ms in the conf) */
+	int idle_ping;				/* MUX idle-ping interval in ms */
 
 	char *id;				/* just for identification */
 	uint32_t rid;				/* revision: if id has been reused for a new server, rid won't match */

src/mux_h2.c

@@ -848,6 +848,9 @@ static void h2c_update_timeout(struct h2c *h2c)
 
 	TRACE_ENTER(H2_EV_H2C_WAKE, h2c->conn);
 
+	/* Always reset flag for PING emission prior to refresh timeout. */
+	h2c->flags &= ~H2_CF_IDL_PING;
+
 	if (!h2c->task)
 		goto leave;
 
@@ -901,9 +904,13 @@ static void h2c_update_timeout(struct h2c *h2c)
 
 					is_idle_conn = 1;
 				}
-				else {
-					/* No timeout on backend idle conn. */
-					exp = TICK_ETERNITY;
+				else if (!(h2c->proxy->flags & (PR_FL_DISABLED|PR_FL_STOPPED))) {
+					/* Only idle-ping is relevant for backend idle conn. */
+					exp = tick_add_ifset(now_ms, conn_idle_ping(h2c->conn));
+					if (tick_isset(exp) && !(h2c->flags & H2_CF_IDL_PING_SENT)) {
+						/* If PING timer selected, set flag to trigger its emission rather than conn deletion on next timeout. */
+						h2c->flags |= H2_CF_IDL_PING;
+					}
 				}
 			}
 
@@ -1308,7 +1315,8 @@ static int h2_init(struct connection *conn, struct proxy *prx, struct session *s
 	h2c->next_tasklet = NULL;
 	h2c->shared_rx_bufs = NULL;
 	h2c->idle_start = now_ms;
-	if (tick_isset(h2c->timeout)) {
+
+	if (tick_isset(h2c->timeout) || tick_isset(conn_idle_ping(conn))) {
 		t = task_new_here();
 		if (!t)
 			goto fail;
@@ -2888,9 +2896,17 @@ static int h2c_ack_settings(struct h2c *h2c)
  */
 static int h2c_handle_ping(struct h2c *h2c)
 {
+	if (h2c->dff & H2_F_PING_ACK) {
+		TRACE_PROTO("receiving H2 PING ACK frame", H2_EV_RX_FRAME|H2_EV_RX_PING, h2c->conn);
+		if ((h2c->flags & (H2_CF_IDL_PING|H2_CF_IDL_PING_SENT)) == H2_CF_IDL_PING_SENT) {
+			h2c->flags &= ~H2_CF_IDL_PING_SENT;
+			h2c_update_timeout(h2c);
+		}
+	}
+	else {
 		/* schedule a response */
-	if (!(h2c->dff & H2_F_PING_ACK))
 		h2c->st0 = H2_CS_FRAME_A;
+	}
 	return 1;
 }
 
@@ -4640,6 +4656,14 @@ static int h2_process_mux(struct h2c *h2c)
 	    h2c_send_conn_wu(h2c) < 0)
 		goto fail;
 
+	/* emit PING to test connection liveliness */
+	if ((h2c->flags & (H2_CF_IDL_PING|H2_CF_IDL_PING_SENT)) == (H2_CF_IDL_PING|H2_CF_IDL_PING_SENT)) {
+		if (!h2c_send_ping(h2c, 0))
+			goto fail;
+		TRACE_USER("sent ping", H2_EV_H2C_WAKE, h2c->conn);
+		h2c->flags &= ~H2_CF_IDL_PING;
+	}
+
 	/* First we always process the flow control list because the streams
 	 * waiting there were already elected for immediate emission but were
 	 * blocked just on this.
@@ -5172,6 +5196,15 @@ struct task *h2_timeout_task(struct task *t, void *context, unsigned int state)
 			return t;
 		}
 
+		if (h2c->flags & H2_CF_IDL_PING) {
+			h2c->flags |= H2_CF_IDL_PING_SENT;
+			tasklet_wakeup(h2c->wait_event.tasklet);
+			TRACE_DEVEL("leaving (idle ping)", H2_EV_H2C_WAKE, h2c->conn);
+			t->expire = conn_idle_ping(h2c->conn);
+			HA_SPIN_UNLOCK(IDLE_CONNS_LOCK, &idle_conns[tid].idle_conns_lock);
+			return t;
+		}
+
 		/* We're about to destroy the connection, so make sure nobody attempts
 		 * to steal it from us.
 		 */
@@ -5469,6 +5502,12 @@ static void h2_detach(struct sedesc *sd)
 
 	if (h2c->flags & H2_CF_IS_BACK) {
 		if (!(h2c->flags & (H2_CF_RCVD_SHUT|H2_CF_ERR_PENDING|H2_CF_ERROR))) {
+			/* Ensure idle-ping is activated before going to idle. */
+			if (eb_is_empty(&h2c->streams_by_id) &&
+			    tick_isset(conn_idle_ping(h2c->conn))) {
+				h2c_update_timeout(h2c);
+			}
+
 			if (h2c->conn->flags & CO_FL_PRIVATE) {
 				/* Add the connection in the session server list, if not already done */
 				if (!session_add_conn(sess, h2c->conn, h2c->conn->target)) {

src/server.c

@@ -1003,6 +1003,43 @@ static int srv_parse_hash_key(char **args, int *cur_arg,
 	return 0;
 }
 
+/* Parse the "idle-ping" server keyword */
+static int srv_parse_idle_ping(char **args, int *cur_arg,
+                               struct proxy *curproxy, struct server *newsrv, char **err)
+{
+	const char *res;
+	unsigned int value;
+
+	if (!*(args[*cur_arg+1])) {
+		memprintf(err, "'%s' expects an argument.", args[*cur_arg]);
+		goto error;
+	}
+
+	res = parse_time_err(args[*cur_arg+1], &value, TIME_UNIT_MS);
+	if (res == PARSE_TIME_OVER) {
+		memprintf(err, "timer overflow in argument <%s> to <%s> of server %s, maximum value is 2147483647 ms (~24.8 days).",
+		          args[*cur_arg+1], args[*cur_arg], newsrv->id);
+		goto error;
+	}
+	else if (res == PARSE_TIME_UNDER) {
+		memprintf(err, "timer underflow in argument <%s> to <%s> of server %s, minimum non-null value is 1 ms.",
+		          args[*cur_arg+1], args[*cur_arg], newsrv->id);
+		goto error;
+	}
+	else if (res) {
+		memprintf(err, "unexpected character '%c' in '%s' argument of server %s.",
+		          *res, args[*cur_arg], newsrv->id);
+		goto error;
+	}
+
+	newsrv->idle_ping = value;
+
+	return 0;
+
+  error:
+	return ERR_ALERT | ERR_FATAL;
+}
+
 /* Parse the "init-addr" server keyword */
 static int srv_parse_init_addr(char **args, int *cur_arg,
                                struct proxy *curproxy, struct server *newsrv, char **err)
@@ -2361,6 +2398,7 @@ static struct srv_kw_list srv_kws = { "ALL", { }, {
 	{ "ws",                   srv_parse_ws,                   1,  1,  1 }, /* websocket protocol */
 	{ "hash-key",             srv_parse_hash_key,             1,  1,  1 }, /* Configure how chash keys are computed */
 	{ "id",                   srv_parse_id,                   1,  0,  1 }, /* set id# of server */
+	{ "idle-ping",            srv_parse_idle_ping,            1,  1,  1 }, /* Activate idle ping if mux support it */
 	{ "init-addr",            srv_parse_init_addr,            1,  1,  0 }, /* */
 	{ "init-state",           srv_parse_init_state,           1,  1,  1 }, /* Set the initial state of the server */
 	{ "log-bufsize",          srv_parse_log_bufsize,          1,  1,  0 }, /* Set the ring bufsize for log server (only for log backends) */