From 381ed2a4dda4e0ddd0bd84cb40e0dd2ae912733b Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Fri, 24 May 2024 09:46:49 +0200 Subject: [PATCH] MINOR: config: add thread-hard-limit to set an upper bound to nbthread On todays large systems, it's not always desired to run on all threads for light loads, and usually users enforce nbthread to a lower value (e.g. 8). The problem is that this is a fixed value, and moving such configs to smaller machines continues to enforce the value and this becomes extremely unproductive due to having more threads than CPUs. This also happens quite a bit in VMs, containers, or cloud instances of various sizes. This commit introduces the thread-hard-limit setting that allows to only set an upper bound to the number of threads without raising a lower value. This means that using "thread-hard-limit 8" will make sure that no more than 8 threads will be used when available, but it will remain two when run on a dual-core machine. --- doc/configuration.txt | 16 +++++++++++++++- include/haproxy/global-t.h | 1 + src/cfgparse.c | 22 ++++++++++++++++++++-- src/thread.c | 30 ++++++++++++++++++++++++++++++ 4 files changed, 66 insertions(+), 3 deletions(-) diff --git a/doc/configuration.txt b/doc/configuration.txt index c0667af8f..3961a1bc7 100644 --- a/doc/configuration.txt +++ b/doc/configuration.txt @@ -2156,7 +2156,8 @@ nbthread bound to upon startup. This means that the thread count can easily be adjusted from the calling process using commands like "taskset" or "cpuset". Otherwise, this value defaults to 1. The default value is reported in the - output of "haproxy -vv". + output of "haproxy -vv". Note that values set here or automatically detected + are subject to the limit set by "thread-hard-limit" (if set). no-quic Disable QUIC transport protocol. All the QUIC listeners will still be created. @@ -2735,6 +2736,19 @@ thread-groups since up to 64 threads per group may be configured. The maximum number of groups is configured at compile time and defaults to 16. See also "nbthread". +thread-hard-limit + This setting is used to enforce a limit to the number of threads, either + detected, or configured. This is particularly useful on operating systems + where the number of threads is automatically detected, where a number of + threads lower than the number of CPUs is desired in generic and portable + configurations. Indeed, while "nbthread" enforces a number of threads that + will result in a warning and bad performance if higher than CPUs available, + thread-hard-limit will only cap the maximum value and automatically limit + the number of threads to no higher than this value, but will not raise lower + values. If "nbthread" is forced to a higher value, thread-hard-limit wins, + and a warning is emitted in so that the configuration anomaly can be + fixed. By default there is no limit. See also "nbthread". + trace This command configures one "trace" subsystem statement. Each of them can be found in the management manual, and follow the exact same syntax. Only one diff --git a/include/haproxy/global-t.h b/include/haproxy/global-t.h index a1357ac43..f38c28eae 100644 --- a/include/haproxy/global-t.h +++ b/include/haproxy/global-t.h @@ -214,6 +214,7 @@ struct global { } unix_bind; struct proxy *cli_fe; /* the frontend holding the stats settings */ int numa_cpu_mapping; + int thread_limit; /* hard limit on the number of threads */ int prealloc_fd; int cfg_curr_line; /* line number currently being parsed */ const char *cfg_curr_file; /* config file currently being parsed or NULL */ diff --git a/src/cfgparse.c b/src/cfgparse.c index c09d96435..f5cde502c 100644 --- a/src/cfgparse.c +++ b/src/cfgparse.c @@ -2730,6 +2730,13 @@ int check_config_validity() if (!global.tune.requri_len) global.tune.requri_len = REQURI_LEN; + if (!global.thread_limit) + global.thread_limit = MAX_THREADS; + +#if defined(USE_THREAD) + if (thread_cpus_enabled_at_boot > global.thread_limit) + thread_cpus_enabled_at_boot = global.thread_limit; +#endif if (!global.nbthread) { /* nbthread not set, thus automatic. In this case, and only if * running on a single process, we enable the same number of @@ -2753,13 +2760,24 @@ int check_config_validity() global.nbtgroups = 1; if (global.nbthread > MAX_THREADS_PER_GROUP * global.nbtgroups) { - ha_diag_warning("nbthread not set, found %d CPUs, limiting to %d threads (maximum is %d per thread group). Please set nbthreads and/or increase thread-groups in the global section to silence this warning.\n", - global.nbthread, MAX_THREADS_PER_GROUP * global.nbtgroups, MAX_THREADS_PER_GROUP); + if (global.nbthread <= global.thread_limit) + ha_diag_warning("nbthread not set, found %d CPUs, limiting to %d threads (maximum is %d per thread group). " + "Please set nbthreads and/or increase thread-groups in the global section to silence this warning.\n", + global.nbthread, MAX_THREADS_PER_GROUP * global.nbtgroups, MAX_THREADS_PER_GROUP); global.nbthread = MAX_THREADS_PER_GROUP * global.nbtgroups; } + + if (global.nbthread > global.thread_limit) + global.nbthread = global.thread_limit; } #endif } + else if (global.nbthread > global.thread_limit) { + ha_warning("nbthread forced to a higher value (%d) than the configured thread-hard-limit (%d), enforcing the limit. " + "Please fix either value to remove this warning.\n", + global.nbthread, global.thread_limit); + global.nbthread = global.thread_limit; + } if (!global.nbtgroups) global.nbtgroups = 1; diff --git a/src/thread.c b/src/thread.c index ab4342dc7..655e1998c 100644 --- a/src/thread.c +++ b/src/thread.c @@ -1709,6 +1709,35 @@ static int cfg_parse_nbthread(char **args, int section_type, struct proxy *curpx return 0; } +/* Parse the "thread-hard-limit" global directive, which takes an integer + * argument that contains the desired maximum number of threads that will + * not be crossed. + */ +static int cfg_parse_thread_hard_limit(char **args, int section_type, struct proxy *curpx, + const struct proxy *defpx, const char *file, int line, + char **err) +{ + long nbthread; + char *errptr; + + if (too_many_args(1, args, err, NULL)) + return -1; + + nbthread = strtol(args[1], &errptr, 10); + if (!*args[1] || *errptr) { + memprintf(err, "'%s' passed a missing or unparsable integer value in '%s'", args[0], args[1]); + return -1; + } + + if (nbthread < 1 || nbthread > MAX_THREADS) { + memprintf(err, "'%s' value must be at least 1 (was %ld)", args[0], nbthread); + return -1; + } + + global.thread_limit = nbthread; + return 0; +} + /* Parse the "thread-group" global directive, which takes an integer argument * that designates a thread group, and a list of threads to put into that group. */ @@ -1855,6 +1884,7 @@ static int cfg_parse_thread_groups(char **args, int section_type, struct proxy * /* config keyword parsers */ static struct cfg_kw_list cfg_kws = {ILH, { + { CFG_GLOBAL, "thread-hard-limit", cfg_parse_thread_hard_limit, 0 }, { CFG_GLOBAL, "nbthread", cfg_parse_nbthread, 0 }, { CFG_GLOBAL, "thread-group", cfg_parse_thread_group, 0 }, { CFG_GLOBAL, "thread-groups", cfg_parse_thread_groups, 0 },