mirror of
https://git.haproxy.org/git/haproxy.git/
synced 2025-08-07 07:37:02 +02:00
A quic_conn is instantiated and tied on the first thread which has received the first INITIAL packet. After handshake completion, listener_accept() is called. For each quic_conn, a new thread is selected among the least loaded ones Note that this occurs earlier if handling 0-RTT data. This thread connection migration is done in two steps : * inside listener_accept(), on the origin thread, quic_conn tasks/tasklet are killed. After this, no quic_conn related processing will occur on this thread. The connection is flagged with QUIC_FL_CONN_AFFINITY_CHANGED. * as soon as the first quic_conn related processing occurs on the new thread, the migration is finalized. This allows to allocate the new tasks/tasklet directly on the destination thread. This last step on the new thread must be done prior to other quic_conn access. There is two events which may trigger it : * a packet is received on the new thread. In this case, qc_finalize_affinity_rebind() is called from quic_dgram_parse(). * the recently accepted connection is popped from accept_queue_ring via accept_queue_process(). This will called session_accept_fd() as listener.bind_conf.accept callback. This instantiates a new session and start connection stack via conn_xprt_start(), which itself calls qc_xprt_start() where qc_finalize_affinity_rebind() is used. A condition was recently found which could cause a closing to be used with qc_finalize_affinity_rebind() which is forbidden with a BUG_ON(). This lat step was not compatible with layer 4 rule such as "tcp-request connection reject" which closes the connection early. In this case, most of the body of session_accept_fd() is skipped, including qc_xprt_start(), so thread migration is not finalized. At the end of the function, conn_xprt_close() is then called which flags the connection as CLOSING. If a datagram is received for this connection before it is released, this will call qc_finalize_affinity_rebind() which triggers its BUG_ON() to prevent thread migration for CLOSING quic_conn. FATAL: bug condition "qc->flags & ((1U << 29)|(1U << 30))" matched at src/quic_conn.c:2036 Thread 3 "haproxy" received signal SIGILL, Illegal instruction. [Switching to Thread 0x7ffff794f700 (LWP 2973030)] 0x00005555556221f3 in qc_finalize_affinity_rebind (qc=0x7ffff002d060) at src/quic_conn.c:2036 2036 BUG_ON(qc->flags & (QUIC_FL_CONN_CLOSING|QUIC_FL_CONN_DRAINING)); (gdb) bt #0 0x00005555556221f3 in qc_finalize_affinity_rebind (qc=0x7ffff002d060) at src/quic_conn.c:2036 #1 0x0000555555682463 in quic_dgram_parse (dgram=0x7fff5003ef10, from_qc=0x0, li=0x555555f38670) at src/quic_rx.c:2602 #2 0x0000555555651aae in quic_lstnr_dghdlr (t=0x555555fc4440, ctx=0x555555fc3f78, state=32832) at src/quic_sock.c:189 #3 0x00005555558c9393 in run_tasks_from_lists (budgets=0x7ffff7944c90) at src/task.c:596 #4 0x00005555558c9e8e in process_runnable_tasks () at src/task.c:876 #5 0x000055555586b7b2 in run_poll_loop () at src/haproxy.c:2966 #6 0x000055555586be87 in run_thread_poll_loop (data=0x555555d3d340 <ha_thread_info+64>) at src/haproxy.c:3165 #7 0x00007ffff7b59609 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #8 0x00007ffff7a7e133 in clone () from /lib/x86_64-linux-gnu/libc.so.6 To fix this issue, ensure quic_conn migration is completed earlier inside session_accept_fd(), before any tcp rules processing. This is done by moving qc_finalize_affinity_rebind() invocation from qc_xprt_start() to qc_conn_init(). This must be backported up to 2.7.
176 lines
4.7 KiB
C
176 lines
4.7 KiB
C
/*
|
|
* QUIC xprt layer. Act as an abstraction between quic_conn and MUX layers.
|
|
*
|
|
* Copyright 2020 HAProxy Technologies, Frederic Lecaille <flecaille@haproxy.com>
|
|
*
|
|
* This program is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU General Public License
|
|
* as published by the Free Software Foundation; either version
|
|
* 2 of the License, or (at your option) any later version.
|
|
*
|
|
*/
|
|
|
|
#include <haproxy/api.h>
|
|
#include <haproxy/connection.h>
|
|
#include <haproxy/quic_conn.h>
|
|
#include <haproxy/ssl_sock.h>
|
|
#include <haproxy/quic_trace.h>
|
|
#include <haproxy/trace.h>
|
|
|
|
static void quic_close(struct connection *conn, void *xprt_ctx)
|
|
{
|
|
struct ssl_sock_ctx *conn_ctx = xprt_ctx;
|
|
struct quic_conn *qc = conn_ctx->qc;
|
|
|
|
TRACE_ENTER(QUIC_EV_CONN_CLOSE, qc);
|
|
|
|
/* Next application data can be dropped. */
|
|
qc->mux_state = QC_MUX_RELEASED;
|
|
|
|
/* If the quic-conn timer has already expired or if already in "connection close"
|
|
* state, free the quic-conn.
|
|
*/
|
|
if (qc->flags & (QUIC_FL_CONN_EXP_TIMER|QUIC_FL_CONN_CLOSING)) {
|
|
quic_conn_release(qc);
|
|
qc = NULL;
|
|
goto leave;
|
|
}
|
|
|
|
/* Schedule a CONNECTION_CLOSE emission. If process stopping is in
|
|
* progress, quic-conn idle-timer will be scheduled immediately after
|
|
* its emission to ensure an immediate connection closing.
|
|
*/
|
|
qc_check_close_on_released_mux(qc);
|
|
leave:
|
|
TRACE_LEAVE(QUIC_EV_CONN_CLOSE, qc);
|
|
}
|
|
|
|
/* Called from the upper layer, to subscribe <es> to events <event_type>. The
|
|
* event subscriber <es> is not allowed to change from a previous call as long
|
|
* as at least one event is still subscribed. The <event_type> must only be a
|
|
* combination of SUB_RETRY_RECV and SUB_RETRY_SEND. It always returns 0.
|
|
*/
|
|
static int quic_conn_subscribe(struct connection *conn, void *xprt_ctx, int event_type, struct wait_event *es)
|
|
{
|
|
struct quic_conn *qc = conn->handle.qc;
|
|
|
|
TRACE_ENTER(QUIC_EV_CONN_SUB, qc);
|
|
|
|
BUG_ON(event_type & ~(SUB_RETRY_SEND|SUB_RETRY_RECV));
|
|
BUG_ON(qc->subs && qc->subs != es);
|
|
|
|
es->events |= event_type;
|
|
qc->subs = es;
|
|
|
|
/* TODO implement a check_events to detect if subscriber should be
|
|
* woken up immediately ?
|
|
*/
|
|
|
|
if (event_type & SUB_RETRY_RECV)
|
|
TRACE_DEVEL("subscribe(recv)", QUIC_EV_CONN_XPRTRECV, qc);
|
|
|
|
if (event_type & SUB_RETRY_SEND)
|
|
TRACE_DEVEL("subscribe(send)", QUIC_EV_CONN_XPRTSEND, qc);
|
|
|
|
TRACE_LEAVE(QUIC_EV_CONN_SUB, qc);
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* Called from the upper layer, to unsubscribe <es> from events <event_type>.
|
|
* The <es> pointer is not allowed to differ from the one passed to the
|
|
* subscribe() call. It always returns zero.
|
|
*/
|
|
static int quic_conn_unsubscribe(struct connection *conn, void *xprt_ctx, int event_type, struct wait_event *es)
|
|
{
|
|
struct quic_conn *qc = conn->handle.qc;
|
|
|
|
TRACE_ENTER(QUIC_EV_CONN_SUB, qc);
|
|
|
|
if (event_type & SUB_RETRY_RECV)
|
|
TRACE_DEVEL("unsubscribe(recv)", QUIC_EV_CONN_XPRTRECV, qc);
|
|
if (event_type & SUB_RETRY_SEND)
|
|
TRACE_DEVEL("unsubscribe(send)", QUIC_EV_CONN_XPRTSEND, qc);
|
|
|
|
es->events &= ~event_type;
|
|
if (!es->events)
|
|
qc->subs = NULL;
|
|
|
|
/* TODO implement ignore_events similar to conn_unsubscribe() ? */
|
|
|
|
TRACE_LEAVE(QUIC_EV_CONN_SUB, qc);
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* Store in <xprt_ctx> the context attached to <conn>.
|
|
* Returns always 0.
|
|
*/
|
|
static int qc_conn_init(struct connection *conn, void **xprt_ctx)
|
|
{
|
|
struct quic_conn *qc = conn->handle.qc;
|
|
|
|
TRACE_ENTER(QUIC_EV_CONN_NEW, qc);
|
|
|
|
/* Ensure thread connection migration is finalized ASAP. */
|
|
if (qc->flags & QUIC_FL_CONN_AFFINITY_CHANGED)
|
|
qc_finalize_affinity_rebind(qc);
|
|
|
|
/* do not store the context if already set */
|
|
if (*xprt_ctx)
|
|
goto out;
|
|
|
|
*xprt_ctx = qc->xprt_ctx;
|
|
|
|
out:
|
|
TRACE_LEAVE(QUIC_EV_CONN_NEW, qc);
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* Start the QUIC transport layer */
|
|
static int qc_xprt_start(struct connection *conn, void *ctx)
|
|
{
|
|
int ret = 0;
|
|
struct quic_conn *qc;
|
|
|
|
qc = conn->handle.qc;
|
|
TRACE_ENTER(QUIC_EV_CONN_NEW, qc);
|
|
|
|
/* mux-quic can now be considered ready. */
|
|
qc->mux_state = QC_MUX_READY;
|
|
|
|
ret = 1;
|
|
out:
|
|
TRACE_LEAVE(QUIC_EV_CONN_NEW, qc);
|
|
return ret;
|
|
}
|
|
|
|
static struct ssl_sock_ctx *qc_get_ssl_sock_ctx(struct connection *conn)
|
|
{
|
|
if (!conn || conn->xprt != xprt_get(XPRT_QUIC) || !conn->handle.qc || !conn->xprt_ctx)
|
|
return NULL;
|
|
|
|
return conn->handle.qc->xprt_ctx;
|
|
}
|
|
|
|
/* transport-layer operations for QUIC connections. */
|
|
static struct xprt_ops ssl_quic = {
|
|
.close = quic_close,
|
|
.subscribe = quic_conn_subscribe,
|
|
.unsubscribe = quic_conn_unsubscribe,
|
|
.init = qc_conn_init,
|
|
.start = qc_xprt_start,
|
|
.prepare_bind_conf = ssl_sock_prepare_bind_conf,
|
|
.destroy_bind_conf = ssl_sock_destroy_bind_conf,
|
|
.get_alpn = ssl_sock_get_alpn,
|
|
.get_ssl_sock_ctx = qc_get_ssl_sock_ctx,
|
|
.name = "QUIC",
|
|
};
|
|
|
|
static void __quic_conn_init(void)
|
|
{
|
|
xprt_register(XPRT_QUIC, &ssl_quic);
|
|
}
|
|
INITCALL0(STG_REGISTER, __quic_conn_init);
|