diff --git a/doc/configuration.txt b/doc/configuration.txt index 866c47e8b..43f25a8c7 100644 --- a/doc/configuration.txt +++ b/doc/configuration.txt @@ -1895,6 +1895,9 @@ The following keywords are supported in the "global" section : - tune.quic.be.cc.reorder-ratio - tune.quic.be.max-idle-timeout - tune.quic.be.sec.glitches-threshold + - tune.quic.be.stream.data-ratio + - tune.quic.be.stream.max-concurrent + - tune.quic.be.stream.rxbuf - tune.quic.be.tx.pacing - tune.quic.be.tx.udp-gso - tune.quic.cc.cubic.min-losses (deprecated) @@ -1910,13 +1913,16 @@ The following keywords are supported in the "global" section : - tune.quic.fe.sec.glitches-threshold - tune.quic.fe.sec.retry-threshold - tune.quic.fe.sock-per-conn + - tune.quic.fe.stream.data-ratio + - tune.quic.fe.stream.max-concurrent + - tune.quic.fe.stream.rxbuf - tune.quic.fe.tx.pacing - tune.quic.fe.tx.udp-gso - - tune.quic.frontend.max-data-size + - tune.quic.frontend.max-data-size (deprecated) - tune.quic.frontend.max-idle-timeout (deprecated) - - tune.quic.frontend.max-streams-bidi + - tune.quic.frontend.max-streams-bidi (deprecated) - tune.quic.frontend.max-tx-mem (deprecated) - - tune.quic.frontend.stream-data-ratio + - tune.quic.frontend.stream-data-ratio (deprecated) - tune.quic.frontend.default-max-window-size (deprecated) - tune.quic.listen - tune.quic.max-frame-loss (deprecated) @@ -4879,6 +4885,74 @@ tune.quic.socket-owner { connection | listener } (deprecated) option is named "tune.quic.fe.sock-per-conn", with legacy value "connection" corresponding to "default-on" and "listener" to "force-off". +tune.quic.be.stream.data-ratio <0..100, in percent> +tune.quic.fe.stream.data-ratio <0..100, in percent> + This setting allows to configure the hard limit of the number of data bytes + in flight over each stream. It is expressed as a percentage relative to the + QUIC stream rxbuf connection setting, with the result rounded up to bufsize. + + The default value is 90. This is suitable with the most frequent web + scenario, where uploads is performed only for one or a few streams, whereas + the rest are used for download only. If the stream rxbuf connection limit + remains at a reasonable level, it ensures that only a portion of opened + streams can allocate to their maximum capacity. + + In the case of an application using many uploading streams in parallel and + suffering from unfairness between these streams, it can make sense to reduce + this ratio, to increase fairness and reduce the per-stream bandwidth. + + See also: "tune.quic.be.stream.rxbuf", "tune.quic.fe.stream.rxbuf", + "tune.quic.be.stream.max-concurrent", + "tune.quic.fe.stream.max-concurrent" + +tune.quic.frontend.stream-data-ratio <0..100, in percent> (deprecated) + This keyword has been deprecated in 3.3 and will be removed in 3.5. It is + part of the streamlining process apply on QUIC configuration. If used, this + setting will only be applied on frontend connections. + +tune.quic.be.stream.max-concurrent +tune.quic.fe.stream.max-concurrent + Sets the QUIC initial_max_streams_bidi transport parameter either on frontend + or backend side. This is the maximum number of bidirectional streams that the + remote peer will be authorized to open concurrently during the connection + lifetime. On frontend side, this limits the number of concurrent HTTP/3 + client requests. + + The default value is 100. Note that if you reduces it, it can restrict the + buffering capabilities of streams on receive, which would result in poor + upload throughput. It can be corrected by increasing the QUIC stream rxbuf + connection setting. + + See also: "tune.quic.be.stream.rxbuf", "tune.quic.fe.stream.rxbuf", + "tune.quic.be.stream.data-ratio", "tune.quic.fe.stream.data-ratio" + +tune.quic.frontend.max-streams-bidi (deprecated) + This keyword has been deprecated in 3.3 and will be removed in 3.5. It is + part of the streamlining process apply on QUIC configuration. If used, this + setting will only be applied on frontend connections. + +tune.quic.be.stream.rxbuf +tune.quic.fe.stream.rxbuf + This setting is the hard limit for the number of data bytes in flight over a + QUIC frontend connection. It is reused as the value for the initial_max_data + transport parameter. It directly impacts the upload bandwidth for the peer + depending on the latency and the per-connection memory consumption in + haproxy. + + By default, the value is set to 0, which indicates that it must be + automatically generated as the product between max-concurrent and bufsize. + This can be increased for example if a backend application relies on massive + uploads over high latency networks. + + See also: "tune.quic.be.stream.max-concurrent", + "tune.quic.fe.stream.max-concurrent", + "tune.quic.be.stream.data-ratio", "tune.quic.fe.stream.data-ratio" + +tune.quic.frontend.max-data-size (deprecated) + This keyword has been deprecated in 3.3 and will be removed in 3.5. It is + part of the streamlining process apply on QUIC configuration. If used, this + setting will only be applied on frontend connections. + tune.quic.be.tx.pacing { on | off } tune.quic.fe.tx.pacing { on | off } Enables ('on') or disables ('off') pacing support for QUIC emission. By @@ -4908,54 +4982,6 @@ tune.quic.disable-udp-gso (deprecated) part of the streamlining process apply on QUIC configuration. If used, this setting will only be applied on frontend connections. -tune.quic.frontend.max-data-size - This setting is the hard limit for the number of data bytes in flight over a - QUIC frontend connection. It is reused as the value for the initial_max_data - transport parameter. It directly impacts the upload bandwidth for the peer - depending on the latency and the per-connection memory consumption in - haproxy. - - By default, the value is set to 0, which indicates that it must be - automatically generated as the product between max-streams-bidi and bufsize. - This can be increased for example if a backend application relies on massive - uploads over high latency networks. - - See also: "tune.quic.frontend.max-streams-bidi", - "tune.quic.frontend.stream-data-ratio" - -tune.quic.frontend.max-streams-bidi - Sets the QUIC initial_max_streams_bidi transport parameter for frontends. - This is the initial maximum number of bidirectional streams the remote peer - will be authorized to open. This determines the number of concurrent client - requests. - - The default value is 100. Note that if you reduces it, it can restrict the - buffering capabilities of streams on receive, which would result in poor - upload throughput. It can be corrected by increasing - tune.quic.frontend.max-data-size setting. - - See also: "tune.quic.frontend.max-data-size", - "tune.quic.frontend.stream-data-ratio" - -tune.quic.frontend.stream-data-ratio <0..100, in percent> - This setting allows to configure the hard limit of the number of data bytes - in flight over each stream. It is expressed as a percentage relative to - "tune.quic.frontend.max-data-size" setting, with the result rounded up to - bufsize. - - The default value is 90. This is suitable with the most frequent web - scenario, where uploads is performed only for one or a few streams, whereas - the rest are used for download only. If max-data-size connection limit - remains at a reasonable level, it ensures that only a portion of opened - streams can allocate to their maximum capacity. - - In the case of an application using many uploading streams in parallel and - suffering from unfairness between these streams, it can make sense to reduce - this ratio, to increase fairness and reduce the per-stream bandwidth. - - See also: "tune.quic.frontend.max-data-size", - "tune.quic.frontend.max-streams-bidi" - tune.quic.listen { on | off } Disable QUIC transport protocol on the frontend side. All the QUIC listeners will still be created, but they won't listen for incoming datagrams. Hence, diff --git a/include/haproxy/global-t.h b/include/haproxy/global-t.h index cc46cd0c1..2e74859ae 100644 --- a/include/haproxy/global-t.h +++ b/include/haproxy/global-t.h @@ -214,11 +214,6 @@ struct global { uint max_checks_per_thread; /* if >0, no more than this concurrent checks per thread */ uint ring_queues; /* if >0, #ring queues, otherwise equals #thread groups */ enum threadgroup_takeover tg_takeover; /* Policy for threadgroup takeover */ -#ifdef USE_QUIC - unsigned int quic_frontend_max_data; - unsigned int quic_frontend_max_streams_bidi; - unsigned int quic_frontend_stream_data_ratio; -#endif /* USE_QUIC */ } tune; struct { char *prefix; /* path prefix of unix bind socket */ diff --git a/include/haproxy/quic_conn-t.h b/include/haproxy/quic_conn-t.h index 2123bbf9f..9ac624a8f 100644 --- a/include/haproxy/quic_conn-t.h +++ b/include/haproxy/quic_conn-t.h @@ -92,9 +92,6 @@ typedef unsigned long long ull; /* Retry token duration */ #define QUIC_RETRY_DURATION_SEC 10 -/* Default ratio applied for max-stream-data-bidi-remote derived from max-data */ -#define QUIC_DFLT_FRONT_STREAM_DATA_RATIO 90 - /* * 0 1 2 3 * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 diff --git a/include/haproxy/quic_tp-t.h b/include/haproxy/quic_tp-t.h index 24c0f1560..569b8041f 100644 --- a/include/haproxy/quic_tp-t.h +++ b/include/haproxy/quic_tp-t.h @@ -41,7 +41,6 @@ struct tp_version_information { /* These ones are our implementation default values when not set * by configuration */ -#define QUIC_TP_DFLT_FRONT_MAX_STREAMS_BIDI 100 #define QUIC_TP_DFLT_FRONT_STREAM_DATA_RATIO 90 /* Types of QUIC transport parameters */ diff --git a/include/haproxy/quic_tune-t.h b/include/haproxy/quic_tune-t.h index e44020e89..2d2675aa6 100644 --- a/include/haproxy/quic_tune-t.h +++ b/include/haproxy/quic_tune-t.h @@ -18,6 +18,11 @@ #define QUIC_DFLT_BE_MAX_IDLE_TIMEOUT 30000 /* milliseconds */ /* Default Retry threshold */ #define QUIC_DFLT_SEC_RETRY_THRESHOLD 100 /* in connection openings */ +/* Default settings related to flow-control */ +#define QUIC_DFLT_FE_STREAM_DATA_RATIO 90 +#define QUIC_DFLT_BE_STREAM_DATA_RATIO 90 +#define QUIC_DFLT_FE_STREAM_MAX_CONCURRENT 100 +#define QUIC_DFLT_BE_STREAM_MAX_CONCURRENT 100 #define QUIC_TUNE_FE_LISTEN_OFF 0x00000001 @@ -36,6 +41,9 @@ struct quic_tune { uint max_idle_timeout; uint sec_glitches_threshold; uint sec_retry_threshold; + uint stream_data_ratio; + uint stream_max_concurrent; + uint stream_rxbuf; uint opts; /* QUIC_TUNE_FE_* options specific to FE side */ uint fb_opts; /* QUIC_TUNE_FB_* options shared by both side */ } fe; @@ -47,6 +55,9 @@ struct quic_tune { uint cc_reorder_ratio; uint max_idle_timeout; uint sec_glitches_threshold; + uint stream_data_ratio; + uint stream_max_concurrent; + uint stream_rxbuf; uint fb_opts; /* QUIC_TUNE_FB_* options shared by both side */ } be; diff --git a/src/cfgparse-quic.c b/src/cfgparse-quic.c index 3dad2b17e..1a1f84047 100644 --- a/src/cfgparse-quic.c +++ b/src/cfgparse-quic.c @@ -30,6 +30,9 @@ struct quic_tune quic_tune = { .cc_reorder_ratio = QUIC_DFLT_CC_REORDER_RATIO, .max_idle_timeout = QUIC_DFLT_FE_MAX_IDLE_TIMEOUT, .sec_retry_threshold = QUIC_DFLT_SEC_RETRY_THRESHOLD, + .stream_data_ratio = QUIC_DFLT_FE_STREAM_DATA_RATIO, + .stream_max_concurrent = QUIC_DFLT_FE_STREAM_MAX_CONCURRENT, + .stream_rxbuf = 0, .fb_opts = QUIC_TUNE_FB_TX_PACING|QUIC_TUNE_FB_TX_UDP_GSO, .opts = QUIC_TUNE_FE_SOCK_PER_CONN, }, @@ -38,6 +41,9 @@ struct quic_tune quic_tune = { .cc_max_win_size = QUIC_DFLT_CC_MAX_WIN_SIZE, .cc_reorder_ratio = QUIC_DFLT_CC_REORDER_RATIO, .max_idle_timeout = QUIC_DFLT_BE_MAX_IDLE_TIMEOUT, + .stream_data_ratio = QUIC_DFLT_BE_STREAM_DATA_RATIO, + .stream_max_concurrent = QUIC_DFLT_BE_STREAM_MAX_CONCURRENT, + .stream_rxbuf = 0, .fb_opts = QUIC_TUNE_FB_TX_PACING|QUIC_TUNE_FB_TX_UDP_GSO, }, .mem_tx_max = QUIC_MAX_TX_MEM, @@ -410,23 +416,33 @@ static int cfg_parse_quic_tune_setting(char **args, int section_type, else if (strcmp(suffix, "fe.sec.retry-threshold") == 0) { quic_tune.fe.sec_retry_threshold = arg; } - else if (strcmp(suffix, "frontend.max-data-size") == 0) { + else if (strcmp(suffix, "be.stream.data-ratio") == 0 || + strcmp(suffix, "fe.stream.data-ratio") == 0) { + uint *ptr = (suffix[0] == 'b') ? &quic_tune.be.stream_data_ratio : + &quic_tune.fe.stream_data_ratio; + if (arg < 1 || arg > 100) { + memprintf(err, "'%s' expects an integer argument between 1 and 100.", args[0]); + return -1; + } + *ptr = arg; + } + else if (strcmp(suffix, "be.stream.max-concurrent") == 0 || + strcmp(suffix, "fe.stream.max-concurrent") == 0) { + uint *ptr = (suffix[0] == 'b') ? &quic_tune.be.stream_max_concurrent : + &quic_tune.fe.stream_max_concurrent; + *ptr = arg; + } + else if (strcmp(suffix, "be.stream.rxbuf") == 0 || + strcmp(suffix, "fe.stream.rxbuf") == 0) { + uint *ptr = (suffix[0] == 'b') ? &quic_tune.be.stream_rxbuf : + &quic_tune.fe.stream_rxbuf; if ((errptr = parse_size_err(args[1], &arg))) { memprintf(err, "'%s': unexpected character '%c' in size argument '%s'.", args[0], *errptr, args[1]); return -1; } - global.tune.quic_frontend_max_data = arg; - } - else if (strcmp(suffix, "frontend.max-streams-bidi") == 0) - global.tune.quic_frontend_max_streams_bidi = arg; - else if (strcmp(suffix, "frontend.stream-data-ratio") == 0) { - if (arg < 1 || arg > 100) { - memprintf(err, "'%s' expects an integer argument between 1 and 100.", args[0]); - return -1; - } - global.tune.quic_frontend_stream_data_ratio = arg; + *ptr = arg; } /* legacy options */ @@ -441,7 +457,7 @@ static int cfg_parse_quic_tune_setting(char **args, int section_type, char *end_opt; memprintf(err, "'%s' is deprecated in 3.3 and will be removed in 3.5. " - "Please use the newer keyword syntax 'tune.quic.fe.cc.max-win-size'.", args[0]); + "Please use the newer keyword syntax 'tune.quic.fe.stream.max-concurrent'.", args[0]); cwnd = parse_window_size(args[0], args[1], &end_opt, err); if (!cwnd) @@ -460,6 +476,24 @@ static int cfg_parse_quic_tune_setting(char **args, int section_type, quic_tune.fe.sec_glitches_threshold = arg; ret = 1; } + else if (strcmp(suffix, "frontend.max-data-size") == 0) { + memprintf(err, "'%s' is deprecated in 3.3 and will be removed in 3.5. " + "Please use the newer keyword syntax 'tune.quic.fe.stream.rxbuf'.", args[0]); + if ((errptr = parse_size_err(args[1], &arg))) { + memprintf(err, "'%s': unexpected character '%c' in size argument '%s'.", + args[0], *errptr, args[1]); + return -1; + } + + quic_tune.fe.stream_rxbuf = arg; + ret = 1; + } + else if (strcmp(suffix, "frontend.max-streams-bidi") == 0) { + memprintf(err, "'%s' is deprecated in 3.3 and will be removed in 3.5. " + "Please use the newer keyword syntax 'tune.quic.fe.stream.max-concurrent'.", args[0]); + quic_tune.fe.stream_max_concurrent = arg; + ret = 1; + } else if (strcmp(suffix, "frontend.max-tx-mem") == 0) { ullong max_mem; @@ -475,6 +509,17 @@ static int cfg_parse_quic_tune_setting(char **args, int section_type, quic_tune.mem_tx_max = max_mem; ret = 1; } + else if (strcmp(suffix, "frontend.stream-data-ratio") == 0) { + memprintf(err, "'%s' is deprecated in 3.3 and will be removed in 3.5. " + "Please use the newer keyword syntax 'tune.quic.fe.stream.data-ratio'.", args[0]); + + if (arg < 1 || arg > 100) { + memprintf(err, "'%s' expects an integer argument between 1 and 100.", args[0]); + return -1; + } + quic_tune.fe.stream_data_ratio = arg; + ret = 1; + } else if (strcmp(suffix, "max-frame-loss") == 0) { memprintf(err, "'%s' is deprecated in 3.3 and will be removed in 3.5. " "Please use the newer keyword syntax 'tune.quic.fe.cc.max-frame-loss'.", args[0]); @@ -617,9 +662,6 @@ static int cfg_parse_quic_tune_on_off(char **args, int section_type, struct prox static struct cfg_kw_list cfg_kws = {ILH, { { CFG_GLOBAL, "tune.quic.listen", cfg_parse_quic_tune_on_off }, { CFG_GLOBAL, "tune.quic.mem.tx-max", cfg_parse_quic_tune_setting }, - { CFG_GLOBAL, "tune.quic.frontend.max-data-size", cfg_parse_quic_tune_setting }, - { CFG_GLOBAL, "tune.quic.frontend.max-streams-bidi", cfg_parse_quic_tune_setting }, - { CFG_GLOBAL, "tune.quic.frontend.stream-data-ratio", cfg_parse_quic_tune_setting }, { CFG_GLOBAL, "tune.quic.zero-copy-fwd-send", cfg_parse_quic_tune_on_off }, { CFG_GLOBAL, "tune.quic.fe.cc.cubic-min-losses", cfg_parse_quic_tune_setting }, @@ -631,6 +673,9 @@ static struct cfg_kw_list cfg_kws = {ILH, { { CFG_GLOBAL, "tune.quic.fe.sec.glitches-threshold", cfg_parse_quic_tune_setting }, { CFG_GLOBAL, "tune.quic.fe.sec.retry-threshold", cfg_parse_quic_tune_setting }, { CFG_GLOBAL, "tune.quic.fe.sock-per-conn", cfg_parse_quic_tune_sock_per_conn }, + { CFG_GLOBAL, "tune.quic.fe.stream.data-ratio", cfg_parse_quic_tune_setting }, + { CFG_GLOBAL, "tune.quic.fe.stream.max-concurrent", cfg_parse_quic_tune_setting }, + { CFG_GLOBAL, "tune.quic.fe.stream.rxbuf", cfg_parse_quic_tune_setting }, { CFG_GLOBAL, "tune.quic.fe.tx.pacing", cfg_parse_quic_tune_on_off }, { CFG_GLOBAL, "tune.quic.fe.tx.udp-gso", cfg_parse_quic_tune_on_off }, @@ -641,6 +686,9 @@ static struct cfg_kw_list cfg_kws = {ILH, { { CFG_GLOBAL, "tune.quic.be.cc.reorder-ratio", cfg_parse_quic_tune_setting }, { CFG_GLOBAL, "tune.quic.be.max-idle-timeout", cfg_parse_quic_time }, { CFG_GLOBAL, "tune.quic.be.sec.glitches-threshold", cfg_parse_quic_tune_setting }, + { CFG_GLOBAL, "tune.quic.be.stream.data-ratio", cfg_parse_quic_tune_setting }, + { CFG_GLOBAL, "tune.quic.be.stream.max-concurrent", cfg_parse_quic_tune_setting }, + { CFG_GLOBAL, "tune.quic.be.stream.rxbuf", cfg_parse_quic_tune_setting }, { CFG_GLOBAL, "tune.quic.be.tx.pacing", cfg_parse_quic_tune_on_off }, { CFG_GLOBAL, "tune.quic.be.tx.udp-gso", cfg_parse_quic_tune_on_off }, @@ -651,8 +699,11 @@ static struct cfg_kw_list cfg_kws = {ILH, { { CFG_GLOBAL, "tune.quic.disable-udp-gso", cfg_parse_quic_tune_setting0 }, { CFG_GLOBAL, "tune.quic.frontend.default-max-window-size", cfg_parse_quic_tune_setting }, { CFG_GLOBAL, "tune.quic.frontend.glitches-threshold", cfg_parse_quic_tune_setting }, + { CFG_GLOBAL, "tune.quic.frontend.max-data-size", cfg_parse_quic_tune_setting }, { CFG_GLOBAL, "tune.quic.frontend.max-idle-timeout", cfg_parse_quic_time }, + { CFG_GLOBAL, "tune.quic.frontend.max-streams-bidi", cfg_parse_quic_tune_setting }, { CFG_GLOBAL, "tune.quic.frontend.max-tx-mem", cfg_parse_quic_tune_setting }, + { CFG_GLOBAL, "tune.quic.frontend.stream-data-ratio", cfg_parse_quic_tune_setting }, { CFG_GLOBAL, "tune.quic.max-frame-loss", cfg_parse_quic_tune_setting }, { CFG_GLOBAL, "tune.quic.reorder-ratio", cfg_parse_quic_tune_setting }, { CFG_GLOBAL, "tune.quic.retry-threshold", cfg_parse_quic_tune_setting }, diff --git a/src/haproxy.c b/src/haproxy.c index d0bf36f0d..8127f0e04 100644 --- a/src/haproxy.c +++ b/src/haproxy.c @@ -100,9 +100,6 @@ #include #include #include -#include -#include -#include #include #include #include @@ -198,11 +195,6 @@ struct global global = { #endif .nb_stk_ctr = MAX_SESS_STKCTR, .default_shards = -2, /* by-group */ -#ifdef USE_QUIC - .quic_frontend_max_data = 0, - .quic_frontend_max_streams_bidi = QUIC_TP_DFLT_FRONT_MAX_STREAMS_BIDI, - .quic_frontend_stream_data_ratio = QUIC_DFLT_FRONT_STREAM_DATA_RATIO, -#endif /* USE_QUIC */ }, #ifdef USE_OPENSSL #ifdef DEFAULT_MAXSSLCONN diff --git a/src/quic_tp.c b/src/quic_tp.c index e85705eee..c8b72c393 100644 --- a/src/quic_tp.c +++ b/src/quic_tp.c @@ -47,7 +47,10 @@ static void quic_dflt_transport_params_cpy(struct quic_transport_params *dst) void quic_transport_params_init(struct quic_transport_params *p, int server) { const uint64_t stream_rx_bufsz = qmux_stream_rx_bufsz(); - const int max_streams_bidi = global.tune.quic_frontend_max_streams_bidi; + const uint stream_rxbuf = server ? + quic_tune.fe.stream_rxbuf : quic_tune.be.stream_rxbuf; + const int max_streams_bidi = server ? + quic_tune.fe.stream_max_concurrent : quic_tune.be.stream_max_concurrent; /* TODO value used to conform with HTTP/3, should be derived from app_ops */ const int max_streams_uni = 3; @@ -70,16 +73,15 @@ void quic_transport_params_init(struct quic_transport_params *p, int server) * or automatically calculated from max number of concurrently opened * streams. */ - if (global.tune.quic_frontend_max_data) - p->initial_max_data = global.tune.quic_frontend_max_data; - else - p->initial_max_data = max_streams_bidi * stream_rx_bufsz; + p->initial_max_data = stream_rxbuf ? + stream_rxbuf : max_streams_bidi * stream_rx_bufsz; /* Set remote streams flow-control data limit. This is calculated as a * ratio from max-data, then rounded up to bufsize. */ - p->initial_max_stream_data_bidi_remote = - p->initial_max_data * global.tune.quic_frontend_stream_data_ratio / 100; + p->initial_max_stream_data_bidi_remote = server ? + p->initial_max_data * quic_tune.fe.stream_data_ratio / 100 : + p->initial_max_data * quic_tune.be.stream_data_ratio / 100; p->initial_max_stream_data_bidi_remote = stream_rx_bufsz * ((p->initial_max_stream_data_bidi_remote + (stream_rx_bufsz - 1)) / stream_rx_bufsz);