From d4e78d873cecf9938885c90e736f8b761a35fb55 Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Thu, 4 Mar 2021 10:47:54 +0100 Subject: [PATCH] MINOR: server: move actconns to the per-thread structure The actconns list creates massive contention on low server counts because it's in fact a list of streams using a server, all threads compete on the list's head and it's still possible to see some watchdog panics on 48 threads under extreme contention with 47 threads trying to add and one thread trying to delete. Moving this list per thread is trivial because it's only used by srv_shutdown_streams(), which simply required to iterate over the list. The field was renamed to "streams" as it's really a list of streams rather than a list of connections. --- include/haproxy/server-t.h | 4 ++-- include/haproxy/stream.h | 2 +- src/cfgparse.c | 1 + src/hlua.c | 2 -- src/server.c | 9 +++++---- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/include/haproxy/server-t.h b/include/haproxy/server-t.h index 37a3bce32..ad3a40f6b 100644 --- a/include/haproxy/server-t.h +++ b/include/haproxy/server-t.h @@ -205,6 +205,7 @@ struct tree_occ { /* Each server will have one occurrence of this structure per thread */ struct srv_per_thread { + struct mt_list streams; /* streams using this server (used by "shutdown server sessions") */ struct eb_root idle_conns; /* Shareable idle connections */ struct eb_root safe_conns; /* Safe idle connections */ struct eb_root avail_conns; /* Connections in use, but with still new streams available */ @@ -236,8 +237,7 @@ struct server { struct be_counters counters; /* statistics counters */ struct eb_root pendconns; /* pending connections */ - struct mt_list actconns; /* active connections (used by "shutdown server sessions") */ - struct srv_per_thread *per_thr; /* array of per-thread stuff such as connections lists, may be null */ + struct srv_per_thread *per_thr; /* array of per-thread stuff such as connections lists */ unsigned int pool_purge_delay; /* Delay before starting to purge the idle conns pool */ unsigned int low_idle_conns; /* min idle connection count to start picking from other threads */ unsigned int max_idle_conns; /* Max number of connection allowed in the orphan connections list */ diff --git a/include/haproxy/stream.h b/include/haproxy/stream.h index 948ca732c..aa6486e25 100644 --- a/include/haproxy/stream.h +++ b/include/haproxy/stream.h @@ -292,7 +292,7 @@ static inline void stream_add_srv_conn(struct stream *sess, struct server *srv) * from a conflict with an adjacent MT_LIST_DEL, and using it improves * the performance by about 3% on 32-cores. */ - MT_LIST_ADD(&srv->actconns, &sess->by_srv); + MT_LIST_ADD(&srv->per_thr[tid].streams, &sess->by_srv); HA_ATOMIC_STORE(&sess->srv_conn, srv); } diff --git a/src/cfgparse.c b/src/cfgparse.c index eafd8ca8e..85995a5d1 100644 --- a/src/cfgparse.c +++ b/src/cfgparse.c @@ -3259,6 +3259,7 @@ out_uri_auth_compat: newsrv->per_thr[i].idle_conns = EB_ROOT; newsrv->per_thr[i].safe_conns = EB_ROOT; newsrv->per_thr[i].avail_conns = EB_ROOT; + MT_LIST_INIT(&newsrv->per_thr[i].streams); } if (newsrv->max_idle_conns != 0) { diff --git a/src/hlua.c b/src/hlua.c index 5e5414c22..12a487267 100644 --- a/src/hlua.c +++ b/src/hlua.c @@ -9175,7 +9175,6 @@ void hlua_init(void) { socket_tcp.next = NULL; socket_tcp.proxy = &socket_proxy; socket_tcp.obj_type = OBJ_TYPE_SERVER; - MT_LIST_INIT(&socket_tcp.actconns); socket_tcp.pendconns = EB_ROOT; LIST_ADD(&servers_list, &socket_tcp.global_list); socket_tcp.next_state = SRV_ST_RUNNING; /* early server setup */ @@ -9221,7 +9220,6 @@ void hlua_init(void) { socket_ssl.next = NULL; socket_ssl.proxy = &socket_proxy; socket_ssl.obj_type = OBJ_TYPE_SERVER; - MT_LIST_INIT(&socket_ssl.actconns); socket_ssl.pendconns = EB_ROOT; LIST_ADD(&servers_list, &socket_ssl.global_list); socket_ssl.next_state = SRV_ST_RUNNING; /* early server setup */ diff --git a/src/server.c b/src/server.c index 5844c6345..04d234779 100644 --- a/src/server.c +++ b/src/server.c @@ -891,10 +891,12 @@ void srv_shutdown_streams(struct server *srv, int why) { struct stream *stream; struct mt_list *elt1, elt2; + int thr; - mt_list_for_each_entry_safe(stream, &srv->actconns, by_srv, elt1, elt2) - if (stream->srv_conn == srv) - stream_shutdown(stream, why); + for (thr = 0; thr < global.nbthread; thr++) + mt_list_for_each_entry_safe(stream, &srv->per_thr[thr].streams, by_srv, elt1, elt2) + if (stream->srv_conn == srv) + stream_shutdown(stream, why); } /* Shutdown all connections of all backup servers of a proxy. The caller must @@ -1750,7 +1752,6 @@ struct server *new_server(struct proxy *proxy) srv->obj_type = OBJ_TYPE_SERVER; srv->proxy = proxy; - MT_LIST_INIT(&srv->actconns); srv->pendconns = EB_ROOT; LIST_ADDQ(&servers_list, &srv->global_list);