MEDIUM: quic: implement poller subscribe on sendto error

On sendto() transient error, prior to this patch sending was simulated
and we relied on retransmission to retry sending. This could hurt
significantly the performance.

Thanks to quic-conn owned socket support, it is now possible to improve
this. On transient error, sending is interrupted and quic-conn socket FD
is subscribed on the poller for sending. When send is possible,
quic_conn_sock_fd_iocb() will be in charge of restart sending.

A consequence of this change is on the return value of qc_send_ppkts().
This function will now return 0 on transient error if quic-conn has its
owned socket. This is used to interrupt sending in the calling function.
The flag QUIC_FL_CONN_TO_KILL must be checked to differentiate a fatal
error from a transient one.

This should be backported up to 2.7.

doc/configuration.txt

@@ -3176,8 +3176,9 @@ tune.quic.socket-owner { listener | connection }
   When default "connection" value is set, a dedicated socket will be allocated
   by every QUIC connections. This option is the preferred one to achieve the
   best performance with a large QUIC traffic. This is also the only way to
-  ensure soft-stop is conducted properly without data loss for QUIC
-  connections. However, this relies on some advanced features from the UDP
+  ensure soft-stop is conducted properly without data loss for QUIC connections
+  and cases of transient errors during sendto() operation are handled
+  efficiently. However, this relies on some advanced features from the UDP
   network stack. If your platform is deemed not compatible, haproxy will
   automatically switch to "listener" mode on startup.
 

src/quic_conn.c

@@ -3436,8 +3436,10 @@ static int qc_prep_pkts(struct quic_conn *qc, struct buffer *buf,
  *
  * This function returns 1 for success. On error, there is several behavior
  * depending on underlying sendto() error :
- * - for a fatal error, 0 is returned and connection is killed.
- * - a transient error is assimilated to a success case with 1 returned.
+ * - for an unrecoverable error, 0 is returned and connection is killed.
+ * - a transient error is handled differently if connection has its owned
+ *   socket. If this is the case, 0 is returned and socket is subscribed on the
+ *   poller. The other case is assimilated to a success case with 1 returned.
  *   Remaining data are purged from the buffer and will eventually be detected
  *   as lost which gives the opportunity to retry sending.
  */
@@ -3481,11 +3483,19 @@ int qc_send_ppkts(struct buffer *buf, struct ssl_sock_ctx *ctx)
 		if (!skip_sendto) {
 			int ret = qc_snd_buf(qc, &tmpbuf, tmpbuf.data, 0);
 			if (ret < 0) {
+				TRACE_ERROR("sendto fatal error", QUIC_EV_CONN_SPPKTS, qc);
 				qc_kill_conn(qc);
 				b_del(buf, buf->data);
 				goto leave;
 			}
 			else if (!ret) {
+				/* Connection owned socket : poller will wake us up when transient error is cleared. */
+				if (qc_test_fd(qc)) {
+					TRACE_ERROR("sendto error, subscribe to poller", QUIC_EV_CONN_SPPKTS, qc);
+					goto leave;
+				}
+
+				/* No connection owned-socket : rely on retransmission to retry sending. */
 				skip_sendto = 1;
 				TRACE_ERROR("sendto error, simulate sending for the rest of data", QUIC_EV_CONN_SPPKTS, qc);
 			}
@@ -4260,6 +4270,7 @@ static int qc_send_app_pkts(struct quic_conn *qc, struct list *frms)
 			break;
 
 		if (!qc_send_ppkts(buf, qc->xprt_ctx)) {
+			if (qc->flags & QUIC_FL_CONN_TO_KILL)
 				qc_txb_release(qc);
 			goto err;
 		}
@@ -4359,6 +4370,7 @@ int qc_send_hdshk_pkts(struct quic_conn *qc, int old_data,
 	}
 
 	if (ret && !qc_send_ppkts(buf, qc->xprt_ctx)) {
+		if (qc->flags & QUIC_FL_CONN_TO_KILL)
 			qc_txb_release(qc);
 		goto out;
 	}
@@ -4665,6 +4677,7 @@ struct task *quic_conn_io_cb(struct task *t, void *context, unsigned int state)
 	}
 
 	if (ret && !qc_send_ppkts(buf, qc->xprt_ctx)) {
+		if (qc->flags & QUIC_FL_CONN_TO_KILL)
 			qc_txb_release(qc);
 		goto out;
 	}

src/quic_sock.c

@@ -492,8 +492,16 @@ static void quic_conn_sock_fd_iocb(int fd)
 
 	TRACE_ENTER(QUIC_EV_CONN_RCV, qc);
 
+	if (fd_send_active(fd) && fd_send_ready(fd)) {
+		TRACE_DEVEL("send ready", QUIC_EV_CONN_RCV, qc);
+		fd_stop_send(fd);
+		tasklet_wakeup_after(NULL, qc->wait_event.tasklet);
+	}
+
+	if (fd_recv_ready(fd)) {
 		tasklet_wakeup_after(NULL, qc->wait_event.tasklet);
 		fd_stop_recv(fd);
+	}
 
 	TRACE_LEAVE(QUIC_EV_CONN_RCV, qc);
 }
@@ -516,6 +524,9 @@ int qc_snd_buf(struct quic_conn *qc, const struct buffer *buf, size_t sz,
 
 	do {
 		if (qc_test_fd(qc)) {
+			if (!fd_send_ready(qc->fd))
+				return 0;
+
 			ret = send(qc->fd, b_peek(buf, b_head_ofs(buf)), sz,
 			           MSG_DONTWAIT | MSG_NOSIGNAL);
 		}
@@ -626,6 +637,8 @@ int qc_snd_buf(struct quic_conn *qc, const struct buffer *buf, size_t sz,
 				HA_ATOMIC_INC(&prx_counters->sendto_err);
 
 			/* transient error */
+			fd_want_send(qc->fd);
+			fd_cant_send(qc->fd);
 			TRACE_PRINTF(TRACE_LEVEL_USER, QUIC_EV_CONN_SPPKTS, qc, 0, 0, 0,
 			             "UDP send failure errno=%d (%s)", errno, strerror(errno));
 			return 0;