MINOR: config: add thread-hard-limit to set an upper bound to nbthread

On todays large systems, it's not always desired to run on all threads
for light loads, and usually users enforce nbthread to a lower value
(e.g. 8). The problem is that this is a fixed value, and moving such
configs to smaller machines continues to enforce the value and this
becomes extremely unproductive due to having more threads than CPUs.
This also happens quite a bit in VMs, containers, or cloud instances
of various sizes.

This commit introduces the thread-hard-limit setting that allows to only
set an upper bound to the number of threads without raising a lower value.
This means that using "thread-hard-limit 8" will make sure that no more
than 8 threads will be used when available, but it will remain two when
run on a dual-core machine.

doc/configuration.txt

@@ -2156,7 +2156,8 @@ nbthread <number>
   bound to upon startup. This means that the thread count can easily be
   adjusted from the calling process using commands like "taskset" or "cpuset".
   Otherwise, this value defaults to 1. The default value is reported in the
-  output of "haproxy -vv".
+  output of "haproxy -vv". Note that values set here or automatically detected
+  are subject to the limit set by "thread-hard-limit" (if set).
 
 no-quic
   Disable QUIC transport protocol. All the QUIC listeners will still be created.
@@ -2735,6 +2736,19 @@ thread-groups <number>
   since up to 64 threads per group may be configured. The maximum number of
   groups is configured at compile time and defaults to 16. See also "nbthread".
 
+thread-hard-limit <number>
+  This setting is used to enforce a limit to the number of threads, either
+  detected, or configured. This is particularly useful on operating systems
+  where the number of threads is automatically detected, where a number of
+  threads lower than the number of CPUs is desired in generic and portable
+  configurations. Indeed, while "nbthread" enforces a number of threads that
+  will result in a warning and bad performance if higher than CPUs available,
+  thread-hard-limit will only cap the maximum value and automatically limit
+  the number of threads to no higher than this value, but will not raise lower
+  values. If "nbthread" is forced to a higher value, thread-hard-limit wins,
+  and a warning is emitted in so that the configuration anomaly can be
+  fixed. By default there is no limit. See also "nbthread".
+
 trace <args...>
   This command configures one "trace" subsystem statement. Each of them can be
   found in the management manual, and follow the exact same syntax. Only one

include/haproxy/global-t.h

@@ -214,6 +214,7 @@ struct global {
 	} unix_bind;
 	struct proxy *cli_fe;           /* the frontend holding the stats settings */
 	int numa_cpu_mapping;
+	int thread_limit;               /* hard limit on the number of threads */
 	int prealloc_fd;
 	int cfg_curr_line;              /* line number currently being parsed */
 	const char *cfg_curr_file;      /* config file currently being parsed or NULL */

src/cfgparse.c

@@ -2730,6 +2730,13 @@ int check_config_validity()
 	if (!global.tune.requri_len)
 		global.tune.requri_len = REQURI_LEN;
 
+	if (!global.thread_limit)
+		global.thread_limit = MAX_THREADS;
+
+#if defined(USE_THREAD)
+	if (thread_cpus_enabled_at_boot > global.thread_limit)
+		thread_cpus_enabled_at_boot = global.thread_limit;
+#endif
 	if (!global.nbthread) {
 		/* nbthread not set, thus automatic. In this case, and only if
 		 * running on a single process, we enable the same number of
@@ -2753,13 +2760,24 @@ int check_config_validity()
 				global.nbtgroups = 1;
 
 			if (global.nbthread > MAX_THREADS_PER_GROUP * global.nbtgroups) {
-				ha_diag_warning("nbthread not set, found %d CPUs, limiting to %d threads (maximum is %d per thread group). Please set nbthreads and/or increase thread-groups in the global section to silence this warning.\n",
-					   global.nbthread, MAX_THREADS_PER_GROUP * global.nbtgroups, MAX_THREADS_PER_GROUP);
+				if (global.nbthread <= global.thread_limit)
+					ha_diag_warning("nbthread not set, found %d CPUs, limiting to %d threads (maximum is %d per thread group). "
+							"Please set nbthreads and/or increase thread-groups in the global section to silence this warning.\n",
+							global.nbthread, MAX_THREADS_PER_GROUP * global.nbtgroups, MAX_THREADS_PER_GROUP);
 				global.nbthread = MAX_THREADS_PER_GROUP * global.nbtgroups;
 			}
+
+			if (global.nbthread > global.thread_limit)
+				global.nbthread = global.thread_limit;
 		}
 #endif
 	}
+	else if (global.nbthread > global.thread_limit) {
+		ha_warning("nbthread forced to a higher value (%d) than the configured thread-hard-limit (%d), enforcing the limit. "
+			   "Please fix either value to remove this warning.\n",
+			   global.nbthread, global.thread_limit);
+		global.nbthread = global.thread_limit;
+	}
 
 	if (!global.nbtgroups)
 		global.nbtgroups = 1;

src/thread.c

@@ -1709,6 +1709,35 @@ static int cfg_parse_nbthread(char **args, int section_type, struct proxy *curpx
 	return 0;
 }
 
+/* Parse the "thread-hard-limit" global directive, which takes an integer
+ * argument that contains the desired maximum number of threads that will
+ * not be crossed.
+ */
+static int cfg_parse_thread_hard_limit(char **args, int section_type, struct proxy *curpx,
+                              const struct proxy *defpx, const char *file, int line,
+                              char **err)
+{
+	long nbthread;
+	char *errptr;
+
+	if (too_many_args(1, args, err, NULL))
+		return -1;
+
+	nbthread = strtol(args[1], &errptr, 10);
+	if (!*args[1] || *errptr) {
+		memprintf(err, "'%s' passed a missing or unparsable integer value in '%s'", args[0], args[1]);
+		return -1;
+	}
+
+	if (nbthread < 1 || nbthread > MAX_THREADS) {
+		memprintf(err, "'%s' value must be at least 1 (was %ld)", args[0], nbthread);
+		return -1;
+	}
+
+	global.thread_limit = nbthread;
+	return 0;
+}
+
 /* Parse the "thread-group" global directive, which takes an integer argument
  * that designates a thread group, and a list of threads to put into that group.
  */
@@ -1855,6 +1884,7 @@ static int cfg_parse_thread_groups(char **args, int section_type, struct proxy *
 
 /* config keyword parsers */
 static struct cfg_kw_list cfg_kws = {ILH, {
+	{ CFG_GLOBAL, "thread-hard-limit", cfg_parse_thread_hard_limit, 0 },
 	{ CFG_GLOBAL, "nbthread",       cfg_parse_nbthread, 0 },
 	{ CFG_GLOBAL, "thread-group",   cfg_parse_thread_group, 0 },
 	{ CFG_GLOBAL, "thread-groups",  cfg_parse_thread_groups, 0 },