From ac78c4fd9d87c93bfb61c9c15182c9c0e38053f9 Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Fri, 17 Mar 2023 16:13:05 +0100 Subject: [PATCH] MINOR: ssl-sock: pass the CO_SFL_MSG_MORE info down the stack Despite having replaced the SSL BIOs to use our own raw_sock layer, we still didn't exploit the CO_SFL_MSG_MORE flag which is pretty useful to avoid sending incomplete packets. It's particularly important for SSL since the extra overhead almost guarantees that each send() will be followed by an incomplete (and often odd-sided) segment. We already have an xprt_st set of flags to pass info to the various layers, so let's just add a new one, SSL_SOCK_SEND_MORE, that is set or cleared during ssl_sock_from_buf() to transfer the knowledge of CO_SFL_MSG_MORE. This way we can recover this information and pass it to raw_sock. This alone is sufficient to increase by ~5-10% the H2 bandwidth over SSL when multiple streams are used in parallel. --- include/haproxy/ssl_sock-t.h | 1 + src/ssl_sock.c | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/include/haproxy/ssl_sock-t.h b/include/haproxy/ssl_sock-t.h index 661c70027..022c57487 100644 --- a/include/haproxy/ssl_sock-t.h +++ b/include/haproxy/ssl_sock-t.h @@ -49,6 +49,7 @@ #define SSL_SOCK_ST_FL_16K_WBFSIZE 0x00000002 #define SSL_SOCK_SEND_UNLIMITED 0x00000004 #define SSL_SOCK_RECV_HEARTBEAT 0x00000008 +#define SSL_SOCK_SEND_MORE 0x00000010 /* set MSG_MORE at lower levels */ /* bits 0xFFFFFF00 are reserved to store verify errors. * The CA en CRT error codes will be stored on 7 bits each diff --git a/src/ssl_sock.c b/src/ssl_sock.c index fa080d67a..5993a72b0 100644 --- a/src/ssl_sock.c +++ b/src/ssl_sock.c @@ -214,6 +214,7 @@ static int ha_ssl_write(BIO *h, const char *buf, int num) { struct buffer tmpbuf; struct ssl_sock_ctx *ctx; + uint flags; int ret; ctx = BIO_get_data(h); @@ -221,7 +222,8 @@ static int ha_ssl_write(BIO *h, const char *buf, int num) tmpbuf.area = (void *)(uintptr_t)buf; tmpbuf.data = num; tmpbuf.head = 0; - ret = ctx->xprt->snd_buf(ctx->conn, ctx->xprt_ctx, &tmpbuf, num, 0); + flags = (ctx->xprt_st & SSL_SOCK_SEND_MORE) ? CO_SFL_MSG_MORE : 0; + ret = ctx->xprt->snd_buf(ctx->conn, ctx->xprt_ctx, &tmpbuf, num, flags); if (ret == 0 && !(ctx->conn->flags & (CO_FL_ERROR | CO_FL_SOCK_WR_SH))) { BIO_set_retry_write(h); ret = -1; @@ -6449,6 +6451,11 @@ static size_t ssl_sock_from_buf(struct connection *conn, void *xprt_ctx, const s ctx->xprt_st |= SSL_SOCK_SEND_UNLIMITED; } + if (try < count || flags & CO_SFL_MSG_MORE) + ctx->xprt_st |= SSL_SOCK_SEND_MORE; + else + ctx->xprt_st &= ~SSL_SOCK_SEND_MORE; + #ifdef SSL_READ_EARLY_DATA_SUCCESS if (!SSL_is_init_finished(ctx->ssl) && conn_is_back(conn)) { unsigned int max_early;