MINOR: quic: properly finalize thread rebinding

When a quic_conn instance is rebinded on a new thread its tasks and
tasklet are destroyed and new ones created. Its socket is also migrated
to a new thread which stop reception on it.

To properly reactivate a quic_conn after rebind, wake up its tasks and
tasklet if they were active before thread rebind. Also reactivate
reading on the socket FD. These operations are implemented on a new
function qc_finalize_affinity_rebind().

This should be backported up to 2.7 after a period of observation.

include/haproxy/quic_conn-t.h

@@ -629,6 +629,7 @@ enum qc_mux_state {
 #define QUIC_FL_CONN_HALF_OPEN_CNT_DECREMENTED   (1U << 11) /* The half-open connection counter was decremented */
 #define QUIC_FL_CONN_HANDSHAKE_SPEED_UP          (1U << 12) /* Handshake speeding up was done */
 #define QUIC_FL_CONN_ACK_TIMER_FIRED             (1U << 13) /* idle timer triggered for acknowledgements */
+#define QUIC_FL_CONN_IO_TO_REQUEUE               (1U << 14) /* IO handler must be requeued on new thread after connection migration */
 #define QUIC_FL_CONN_TO_KILL                     (1U << 24) /* Unusable connection, to be killed */
 #define QUIC_FL_CONN_TX_TP_RECEIVED              (1U << 25) /* Peer transport parameters have been received (used for the transmitting part) */
 #define QUIC_FL_CONN_FINALIZED                   (1U << 26) /* QUIC connection finalized (functional, ready to send/receive) */

include/haproxy/quic_conn.h

@@ -701,6 +701,7 @@ static inline void quic_handle_stopping(void)
 }
 
 int qc_set_tid_affinity(struct quic_conn *qc, uint tid);
+void qc_finalize_affinity_rebind(struct quic_conn *qc);
 
 #endif /* USE_QUIC */
 #endif /* _HAPROXY_QUIC_CONN_H */

include/haproxy/quic_sock.h

@@ -65,6 +65,7 @@ static inline char qc_test_fd(struct quic_conn *qc)
 void qc_alloc_fd(struct quic_conn *qc, const struct sockaddr_storage *src,
                  const struct sockaddr_storage *dst);
 void qc_release_fd(struct quic_conn *qc, int reinit);
+void qc_want_recv(struct quic_conn *qc);
 
 void quic_accept_push_qc(struct quic_conn *qc);
 

src/quic_conn.c

@@ -8481,6 +8481,8 @@ int qc_set_tid_affinity(struct quic_conn *qc, uint new_tid)
 	}
 
 	/* Reinit IO tasklet. */
+	if (qc->wait_event.tasklet->state & TASK_IN_LIST)
+		qc->flags |= QUIC_FL_CONN_IO_TO_REQUEUE;
 	tasklet_kill(qc->wait_event.tasklet);
 	/* In most cases quic_conn_app_io_cb is used but for 0-RTT quic_conn_io_cb can be still activated. */
 	t3->process = qc->wait_event.tasklet->process;
@@ -8491,8 +8493,8 @@ int qc_set_tid_affinity(struct quic_conn *qc, uint new_tid)
 
 	/* Rebind the connection FD. */
 	if (qc_test_fd(qc)) {
+		/* Reading is reactivated by the new thread. */
 		fd_migrate_on(qc->fd, new_tid);
-		/* TODO need to reactivate reading on the new thread. */
 	}
 
 	/* Remove conn from per-thread list instance. */
@@ -8524,6 +8526,29 @@ int qc_set_tid_affinity(struct quic_conn *qc, uint new_tid)
 	return 1;
 }
 
+/* Must be called after qc_set_tid_affinity() on the new thread. */
+void qc_finalize_affinity_rebind(struct quic_conn *qc)
+{
+	TRACE_ENTER(QUIC_EV_CONN_SET_AFFINITY, qc);
+
+	/* Reactivate FD polling if connection socket is active. */
+	qc_want_recv(qc);
+
+	/* Reactivate timer task if needed. */
+	qc_set_timer(qc);
+
+	/* Idle timer task is always active. */
+	task_queue(qc->idle_timer_task);
+
+	/* Reactivate IO tasklet if needed. */
+	if (qc->flags & QUIC_FL_CONN_IO_TO_REQUEUE) {
+		tasklet_wakeup(qc->wait_event.tasklet);
+		qc->flags &= ~QUIC_FL_CONN_IO_TO_REQUEUE;
+	}
+
+	TRACE_LEAVE(QUIC_EV_CONN_SET_AFFINITY, qc);
+}
+
 /* appctx context used by "show quic" command */
 struct show_quic_ctx {
 	unsigned int epoch;

src/quic_sock.c

@@ -878,6 +878,15 @@ void qc_release_fd(struct quic_conn *qc, int reinit)
 	}
 }
 
+/* Wrapper for fd_want_recv(). Safe even if connection does not used its owned
+ * socket.
+ */
+void qc_want_recv(struct quic_conn *qc)
+{
+	if (qc_test_fd(qc))
+		fd_want_recv(qc->fd);
+}
+
 /*********************** QUIC accept queue management ***********************/
 /* per-thread accept queues */
 struct quic_accept_queue *quic_accept_queues;