MEDIUM: quic: limit global Tx memory

Define a new settings tune.quic.frontend.max-tot-window. It contains a
size argument which can be used to set a limit on the sum of all QUIC
connections congestion window. This is applied both on
quic_cc_path_set() and quic_cc_path_inc().

Note that this limitation cannot reduce a congestion window more than
the minimal limit which is set to 2 datagrams.

doc/configuration.txt

@@ -1697,6 +1697,7 @@ The following keywords are supported in the "global" section :
    - tune.quic.frontend.max-data-size
    - tune.quic.frontend.max-idle-timeout
    - tune.quic.frontend.max-streams-bidi
+   - tune.quic.frontend.max-tx-mem
    - tune.quic.frontend.stream-data-ratio
    - tune.quic.frontend.default-max-window-size
    - tune.quic.max-frame-loss
@@ -4446,6 +4447,14 @@ tune.quic.frontend.max-streams-bidi <number>
   See also: "tune.quic.frontend.max-data-size",
             "tune.quic.frontend.stream-data-ratio"
 
+tune.quic.frontend.max-tx-mem <size>
+  Sets the maximum amount of memory usable by QUIC stack at the transport layer
+  for emission. This serves both as a limit of in flight bytes and multiplexer
+  output buffers. Note that to prevent threads contention this limit is not
+  strictly enforced so it can be exceeded on some occasions. Also, each
+  connection will always be able to use a window of at least 2 datagrams, so a
+  proper maxconn should be used in conjunction.
+
 tune.quic.frontend.stream-data-ratio  <0..100, in percent>
   This setting allows to configure the hard limit of the number of data bytes
   in flight over each stream. It is expressed as a percentage relative to

include/haproxy/defaults.h

@@ -645,4 +645,13 @@
 #define FWLC_MIN_FREE_ENTRIES 500
 #endif /* FWLC_MIN_FREE_ENTRIES */
 
+/*
+ * QUIC
+ */
+
+/* Memory usage in bytes on Tx side, 0 for unlimited. */
+#ifndef QUIC_MAX_TX_MEM
+#define QUIC_MAX_TX_MEM 0
+#endif
+
 #endif /* _HAPROXY_DEFAULTS_H */

include/haproxy/global-t.h

@@ -217,6 +217,7 @@ struct global {
 		unsigned int quic_frontend_glitches_threshold;
 		unsigned int quic_frontend_max_data;
 		unsigned int quic_frontend_max_streams_bidi;
+		uint64_t quic_frontend_max_tx_mem;
 		size_t quic_frontend_max_window_size;
 		unsigned int quic_frontend_stream_data_ratio;
 		unsigned int quic_retry_threshold;

src/cfgparse-quic.c

@@ -316,6 +316,17 @@ static int cfg_parse_quic_tune_setting(char **args, int section_type,
 	}
 	else if (strcmp(suffix, "frontend.max-streams-bidi") == 0)
 		global.tune.quic_frontend_max_streams_bidi = arg;
+	else if (strcmp(suffix, "frontend.max-tx-mem") == 0) {
+		ullong max_mem;
+
+		if ((errptr = parse_size_err(args[1], &max_mem))) {
+			memprintf(err, "'%s': unexpected character '%c' in size argument '%s'.",
+			          args[0], *errptr, args[1]);
+			return -1;
+		}
+
+		global.tune.quic_frontend_max_tx_mem = max_mem;
+	}
 	else if (strcmp(suffix, "frontend.default-max-window-size") == 0) {
 		unsigned long cwnd;
 		char *end_opt;
@@ -430,6 +441,7 @@ static struct cfg_kw_list cfg_kws = {ILH, {
 	{ CFG_GLOBAL, "tune.quic.frontend.max-data-size", cfg_parse_quic_tune_setting },
 	{ CFG_GLOBAL, "tune.quic.frontend.max-streams-bidi", cfg_parse_quic_tune_setting },
 	{ CFG_GLOBAL, "tune.quic.frontend.max-idle-timeout", cfg_parse_quic_time },
+	{ CFG_GLOBAL, "tune.quic.frontend.max-tx-mem", cfg_parse_quic_tune_setting },
 	{ CFG_GLOBAL, "tune.quic.frontend.default-max-window-size", cfg_parse_quic_tune_setting },
 	{ CFG_GLOBAL, "tune.quic.frontend.stream-data-ratio", cfg_parse_quic_tune_setting },
 	{ CFG_GLOBAL, "tune.quic.max-frame-loss", cfg_parse_quic_tune_setting },

src/haproxy.c

@@ -199,6 +199,7 @@ struct global global = {
 		.quic_frontend_max_idle_timeout = QUIC_TP_DFLT_FRONT_MAX_IDLE_TIMEOUT,
 		.quic_frontend_max_data = 0,
 		.quic_frontend_max_streams_bidi = QUIC_TP_DFLT_FRONT_MAX_STREAMS_BIDI,
+		.quic_frontend_max_tx_mem = QUIC_MAX_TX_MEM,
 		.quic_frontend_max_window_size = QUIC_DFLT_MAX_WINDOW_SIZE,
 		.quic_frontend_stream_data_ratio = QUIC_DFLT_FRONT_STREAM_DATA_RATIO,
 		.quic_reorder_ratio = QUIC_DFLT_REORDER_RATIO,

src/quic_cc.c

@@ -78,6 +78,22 @@ static int quic_cwnd_may_increase(const struct quic_cc_path *path)
 	return 2 * path->in_flight >= path->cwnd  || path->cwnd < 16384;
 }
 
+/* Calculate ratio of free memory relative to the maximum configured limit. */
+static int quic_cc_max_win_ratio(void)
+{
+	uint64_t tot, free = 0;
+	int ratio = 100;
+
+	if (global.tune.quic_frontend_max_tx_mem) {
+		tot = cshared_read(&quic_mem_diff);
+		if (global.tune.quic_frontend_max_tx_mem > tot)
+			free = global.tune.quic_frontend_max_tx_mem - tot;
+		ratio = free * 100 / global.tune.quic_frontend_max_tx_mem;
+	}
+
+	return ratio;
+}
+
 /* Restore congestion window for <path> to its minimal value. */
 void quic_cc_path_reset(struct quic_cc_path *path)
 {
@@ -90,8 +106,9 @@ void quic_cc_path_reset(struct quic_cc_path *path)
 void quic_cc_path_set(struct quic_cc_path *path, uint64_t val)
 {
 	const uint64_t old = path->cwnd;
+	const uint64_t limit_max = path->limit_max * quic_cc_max_win_ratio() / 100;
 
-	path->cwnd = QUIC_MIN(val, path->limit_max);
+	path->cwnd = QUIC_MIN(val, limit_max);
 	path->cwnd = QUIC_MAX(path->cwnd, path->limit_min);
 	cshared_add(&quic_mem_diff, path->cwnd - old);
 
@@ -105,9 +122,11 @@ void quic_cc_path_set(struct quic_cc_path *path, uint64_t val)
 void quic_cc_path_inc(struct quic_cc_path *path, uint64_t val)
 {
 	const uint64_t old = path->cwnd;
+	uint64_t limit_max;
 
 	if (quic_cwnd_may_increase(path)) {
-		path->cwnd = QUIC_MIN(path->cwnd + val, path->limit_max);
+		limit_max = path->limit_max * quic_cc_max_win_ratio() / 100;
+		path->cwnd = QUIC_MIN(path->cwnd + val, limit_max);
 		path->cwnd = QUIC_MAX(path->cwnd, path->limit_min);
 		cshared_add(&quic_mem_diff, path->cwnd - old);