MEDIUM: tcp: add the "tfo" option to support TCP fastopen on the server

This implements support for the new API which relies on a call to
setsockopt().
On systems that support it (currently, only Linux >= 4.11), this enables
using TCP fast open when connecting to server.
Please note that you should use the retry-on "conn-failure", "empty-response"
and "response-timeout" keywords, or the request won't be able to be retried
on failure.

Co-authored-by: Olivier Houchard <ohouchard@haproxy.com>

doc/configuration.txt

@@ -12506,6 +12506,14 @@ tcp-ut <delay>
   argument is a delay expressed in milliseconds by default. This only works for
   regular TCP connections, and is ignored for other protocols.
 
+tfo
+  This option enables using TCP fast open when connecting to servers, on
+  systems that support it (currently only the Linux kernel >= 4.11).
+  See the "tfo" bind option for more information about TCP fast open.
+  Please note that when using tfo, you should also use the "conn-failure",
+  "empty-response" and "response-timeout" keywords for "retry-on", or haproxy
+  won't be able to retry the connection on failure.
+
 track [<proxy>/]<server>
   This option enables ability to set the current state of the server by tracking
   another one. It is possible to track a server which itself tracks another

include/common/compat.h

@@ -128,6 +128,10 @@
 #ifndef TCP_FASTOPEN
 #define TCP_FASTOPEN 23
 #endif
+
+#ifndef TCP_FASTOPEN_CONNECT
+#define TCP_FASTOPEN_CONNECT 30
+#endif
 #endif
 
 /* FreeBSD doesn't define SOL_IP and prefers IPPROTO_IP */

include/proto/stream_interface.h

@@ -484,13 +484,17 @@ static inline void si_chk_snd(struct stream_interface *si)
 static inline int si_connect(struct stream_interface *si, struct connection *conn)
 {
 	int ret = SF_ERR_NONE;
+	int conn_flags = 0;
 
 	if (unlikely(!conn || !conn->ctrl || !conn->ctrl->connect))
 		return SF_ERR_INTERNAL;
 
+	if (!channel_is_empty(si_oc(si)))
+		conn_flags |= CONNECT_HAS_DATA;
+	if (si->conn_retries == si_strm(si)->be->conn_retries)
+		conn_flags |= CONNECT_CAN_USE_TFO;
 	if (!conn_ctrl_ready(conn) || !conn_xprt_ready(conn)) {
-		ret = conn->ctrl->connect(conn, channel_is_empty(si_oc(si)) ?
-		                          CONNECT_HAS_DATA : 0);
+		ret = conn->ctrl->connect(conn, conn_flags);
 		if (ret != SF_ERR_NONE)
 			return ret;
 

include/types/protocol.h

@@ -88,6 +88,7 @@ struct protocol {
 #define CONNECT_HAS_DATA                        0x00000001 /* There's data available to be sent */
 #define CONNECT_DELACK_SMART_CONNECT            0x00000002 /* Use a delayed ACK if the backend has tcp-smart-connect */
 #define CONNECT_DELACK_ALWAYS                   0x00000004 /* Use a delayed ACK */
+#define CONNECT_CAN_USE_TFO                     0x00000008 /* We can use TFO for this connection */
 #endif /* _TYPES_PROTOCOL_H */
 
 /*

include/types/server.h

@@ -143,6 +143,7 @@ enum srv_initaddr {
 #define SRV_F_CHECKPORT    0x0040        /* this server has a check port configured */
 #define SRV_F_AGENTADDR    0x0080        /* this server has a agent addr configured */
 #define SRV_F_COOKIESET    0x0100        /* this server has a cookie configured, so don't generate dynamic cookies */
+#define SRV_F_FASTOPEN     0x0100        /* Use TCP Fast Open to connect to server */
 
 /* configured server options for send-proxy (server->pp_opts) */
 #define SRV_PP_V1               0x0001   /* proxy protocol version 1 */

src/cfgparse.c

@@ -3121,6 +3121,14 @@ int check_config_validity()
 					cfgerr += xprt_get(XPRT_SSL)->prepare_srv(newsrv);
 			}
 
+			if ((newsrv->flags & SRV_F_FASTOPEN) &&
+			    ((curproxy->retry_type & (PR_RE_DISCONNECTED | PR_RE_TIMEOUT)) !=
+			     (PR_RE_DISCONNECTED | PR_RE_TIMEOUT)))
+				ha_warning("parsing [%s:%d] : %s '%s': server '%s' has tfo activated, the backend should be configured with at least 'conn-failure', 'empty-response' and 'response-timeout' or we wouldn't be able to retry the connection on failure.\n",
+				    newsrv->conf.file, newsrv->conf.line,
+				    proxy_type_str(curproxy), curproxy->id,
+				    newsrv->id);
+
 			/* set the check type on the server */
 			newsrv->check.type = curproxy->options2 & PR_O2_CHK_ANY;
 

src/proto_tcp.c

@@ -293,6 +293,7 @@ int tcp_connect_server(struct connection *conn, int flags)
 	struct server *srv;
 	struct proxy *be;
 	struct conn_src *src;
+	int use_fastopen = 0;
 
 	conn->flags |= CO_FL_WAIT_L4_CONN; /* connection in progress */
 
@@ -304,6 +305,14 @@ int tcp_connect_server(struct connection *conn, int flags)
 	case OBJ_TYPE_SERVER:
 		srv = objt_server(conn->target);
 		be = srv->proxy;
+		/* Make sure we check that we have data before activating
+		 * TFO, or we could trigger a kernel issue whereby after
+		 * a successful connect() == 0, any subsequent connect()
+		 * will return EINPROGRESS instead of EISCONN.
+		 */
+		use_fastopen = (srv->flags & SRV_F_FASTOPEN) &&
+		               ((flags & (CONNECT_CAN_USE_TFO | CONNECT_HAS_DATA)) ==
+				(CONNECT_CAN_USE_TFO | CONNECT_HAS_DATA));
 		break;
 	default:
 		conn->flags |= CO_FL_ERROR;
@@ -493,6 +502,12 @@ int tcp_connect_server(struct connection *conn, int flags)
 	if (srv && srv->tcp_ut)
 		setsockopt(fd, IPPROTO_TCP, TCP_USER_TIMEOUT, &srv->tcp_ut, sizeof(srv->tcp_ut));
 #endif
+
+	if (use_fastopen) {
+#if defined(TCP_FASTOPEN_CONNECT)
+                setsockopt(fd, IPPROTO_TCP, TCP_FASTOPEN_CONNECT, &one, sizeof(one));
+#endif
+	}
 	if (global.tune.server_sndbuf)
                 setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &global.tune.server_sndbuf, sizeof(global.tune.server_sndbuf));
 

src/raw_sock.c

@@ -402,7 +402,7 @@ static size_t raw_sock_from_buf(struct connection *conn, void *xprt_ctx, const s
 			if (ret < try)
 				break;
 		}
-		else if (ret == 0 || errno == EAGAIN || errno == ENOTCONN) {
+		else if (ret == 0 || errno == EAGAIN || errno == ENOTCONN || errno == EINPROGRESS) {
 			/* nothing written, we need to poll for write first */
 			fd_cant_send(conn->handle.fd);
 			break;

src/server.c

@@ -889,6 +889,13 @@ static int srv_parse_track(char **args, int *cur_arg,
 }
 
 
+/* parse the "tfo" server keyword */
+static int srv_parse_tfo(char **args, int *cur_arg, struct proxy *px, struct server *newsrv, char **err)
+{
+	newsrv->flags |= SRV_F_FASTOPEN;
+	return 0;
+}
+
 /* Shutdown all connections of a server. The caller must pass a termination
  * code in <why>, which must be one of SF_ERR_* indicating the reason for the
  * shutdown.
@@ -1277,6 +1284,7 @@ static struct srv_kw_list srv_kws = { "ALL", { }, {
 	{ "send-proxy-v2",       srv_parse_send_proxy_v2,       0,  1 }, /* Enforce use of PROXY V2 protocol */
 	{ "source",              srv_parse_source,             -1,  1 }, /* Set the source address to be used to connect to the server */
 	{ "stick",               srv_parse_stick,               0,  1 }, /* Enable stick-table persistence */
+	{ "tfo",                 srv_parse_tfo,                 0,  0 }, /* enable TCP Fast Open of server */
 	{ "track",               srv_parse_track,               1,  1 }, /* Set the current state of the server, tracking another one */
 	{ NULL, NULL, 0 },
 }};