MEDIUM: cli: Add support for dynamically allocated payloads

It is now possible to deal with too big payload to fit in a buffer, without
changing the buffer size. By default, a payload up to 128 KB can be
dynamically allocated. "tune.cli.max-payload-size" global parameter can be
used to change this value, with some caution for huge values.

For CLI command handler functions, there is no change at all. A pointer on
the payload is still passed as parameter. Internally, an area is allocated
for the payload only if it is too big.

The payload pattern used to detect the end of the payload is part from the
allocated area.

doc/configuration.txt

@@ -1884,6 +1884,7 @@ The following keywords are supported in the "global" section :
    - tune.bufsize
    - tune.bufsize.large
    - tune.bufsize.small
+   - tune.cli.max-payload-size
    - tune.comp.maxlevel
    - tune.defaults.purge
    - tune.disable-fast-forward
@@ -4192,6 +4193,22 @@ tune.bufsize.small <size>
 
   See also: option use-small-buffers
 
+tune.cli.max-payload-size <size>
+  Sets the maximum size allowed for the payload passed to a command on the CLI.
+
+  On the CLI, a command line is limited by the buffer size. It means all
+  commands and their arguments must fit in a buffer to be processed, excluding
+  the payload that can be passed to the last command of the command line. This
+  payload can be allocated into a dedicated area if necessary. Its size is
+  limited by this parameter. The default value is 128KB.
+
+  While it should be high enough for most usage, if this value is changed, it
+  must be carefully chosen. A huge value can have impact on the HAProxy
+  performance. Depending on the command, a huge payload can be quite long to
+  process and can possibly trigger the watchdog.
+
+  Please consult the management manual for details about the CLI.
+
 tune.comp.maxlevel <number>
   Sets the maximum compression level. The compression level affects CPU
   usage during compression. This value affects CPU usage during compression.

doc/management.txt

@@ -1647,15 +1647,18 @@ a payload, it needs to end with an empty line.
 The payload pattern can be customized in order to change the way the payload
 ends. In order to end a payload with something else than an empty line, a
 customized pattern can be set between '<<' and '\n'. Only 7 characters can be
-used in addiction to '<<', otherwise this won't be considered a payload.
+used in addition to '<<', otherwise this won't be considered a payload.
 For example, to use a PEM file that contains empty lines and comments:
 
   # echo -e "set ssl cert common.pem <<%EOF%\n$(cat common.pem)\n%EOF%\n" | \
   socat /var/run/haproxy.stat -
 
-Limitations do exist: the length of the whole buffer passed to the CLI must
-not be greater than tune.bfsize and the pattern "<<" must not be glued to the
-last word of the line.
+Limitations do exist: The pattern "<<" must not be glued to the last word of the
+line. The length of a command line must not be greater than tune.bufsize,
+including the pattern starting the payload, but excluding the payload
+itself. The payload size is limited to 128KB by default. This can be changed by
+setting "tune.cli.max-payload-size" global parameter, with some cautions. Note
+the pattern marking the end of the payload is part of this limit.
 
 When entering a payload while in interactive mode, the prompt will change from
 "> " to "+ ".

include/haproxy/applet-t.h

@@ -125,8 +125,8 @@ struct appctx {
 		int severity_output;    /* used within the cli_io_handler to format severity output of informational feedback */
 		int level;              /* the level of CLI which can be lowered dynamically */
 		char *payload_pat;      /* Pointer on the payload pattern. NULL if no payload */
+		uint32_t max_payload_sz;/* Max size allowed for dynamic payload. 0 if not allowed */
 		uint32_t anon_key;      /* the key to anonymise with the hash in cli */
-		/* XXX 4 unused bytes here */
 		int (*io_handler)(struct appctx *appctx);  /* used within the cli_io_handler when st0 = CLI_ST_CALLBACK */
 		void (*io_release)(struct appctx *appctx); /* used within the cli_io_handler when st0 = CLI_ST_CALLBACK,
 							      if the command is terminated or the session released */

include/haproxy/cli-t.h

@@ -49,6 +49,7 @@
 #define APPCTX_CLI_ST1_PROMPT  (1 << 4) /* display prompt */
 #define APPCTX_CLI_ST1_TIMED   (1 << 5) /* display timer in prompt */
 #define APPCTX_CLI_ST1_YIELD   (1 << 6) /* forced yield between commands */
+#define APPCTX_CLI_ST1_DYN_PAYLOAD (1 << 7) /* the payload was dynamically allocated  */
 
 #define CLI_PREFIX_KW_NB 5
 #define CLI_MAX_MATCHES 5

include/haproxy/global-t.h

@@ -215,6 +215,7 @@ struct global {
 		int default_shards; /* default shards for listeners, or -1 (by-thread) or -2 (by-group) */
 		uint max_checks_per_thread; /* if >0, no more than this concurrent checks per thread */
 		uint ring_queues;   /* if >0, #ring queues, otherwise equals #thread groups */
+		uint cli_max_payload_sz; /* The max payload size for the CLI */
 		enum threadgroup_takeover tg_takeover; /* Policy for threadgroup takeover */
 	} tune;
 	struct {

src/cli.c

@@ -653,6 +653,34 @@ static int cli_parse_global(char **args, int section_type, struct proxy *curpx,
 	return 0;
 }
 
+/* This function parses "tune.cli.max-payload-sze" statement in the "global"
+ * section. It returns -1 if there is any error, otherwise zero. If it returns
+ * -1, it will write an error message into the <err> buffer which will be
+ * preallocated. The trailing '\n' must not be written. The function must be
+ * called with <args> pointing to the first word after "stats".
+ */
+static int cli_parse_global_max_payload_size(char **args, int section_type, struct proxy *curpx,
+					     const struct proxy *defpx, const char *file, int line,
+					     char **err)
+{
+	const char *res;
+
+	if (too_many_args(1, args, err, NULL))
+		return -1;
+
+	if (*(args[1]) == 0) {
+		memprintf(err, "'%s' expects a size argument.", args[0]);
+		return -1;
+	}
+	res = parse_size_err(args[1], &global.tune.cli_max_payload_sz);
+	if (res != NULL) {
+		memprintf(err, "unexpected '%s' after size passed to '%s'", res, args[0]);
+		return -1;
+	}
+	return 0;
+}
+
+
 /*
  * This function exports the bound addresses of a <frontend> in the environment
  * variable <varname>. Those addresses are separated by semicolons and prefixed
@@ -950,6 +978,7 @@ int cli_init(struct appctx *appctx)
 	appctx->cli_ctx.payload_pat = NULL;
 	appctx->cli_ctx.cmdline = NULL;
 	appctx->cli_ctx.payload = BUF_NULL;
+	appctx->cli_ctx.max_payload_sz = global.tune.cli_max_payload_sz;
 
 	/* Wakeup the applet ASAP. */
         applet_need_more_data(appctx);
@@ -957,6 +986,29 @@ int cli_init(struct appctx *appctx)
 
 }
 
+int cli_try_realloc_payload(struct appctx *appctx, struct buffer *buf, size_t new_size)
+{
+	char *old, *new;
+
+	if (new_size > appctx->cli_ctx.max_payload_sz)
+		new_size = appctx->cli_ctx.max_payload_sz;
+
+	old = (appctx->st1 & APPCTX_CLI_ST1_DYN_PAYLOAD) ? b_orig(buf) : NULL;
+	new = my_realloc2(old, new_size);
+	if (!new) {
+		*buf = BUF_NULL;
+		appctx->st1 &= ~APPCTX_CLI_ST1_DYN_PAYLOAD;
+		return -1;
+	}
+
+	if (!(appctx->st1 & APPCTX_CLI_ST1_DYN_PAYLOAD)) {
+		memcpy(new, b_orig(buf), b_data(buf));
+		appctx->st1 |= APPCTX_CLI_ST1_DYN_PAYLOAD;
+	}
+	*buf = b_make(new, new_size, 0, b_data(buf));
+	return 0;
+}
+
 int cli_parse_cmdline(struct appctx *appctx)
 {
 	char *str;
@@ -971,9 +1023,7 @@ int cli_parse_cmdline(struct appctx *appctx)
 		appctx->cli_ctx.cmdline = alloc_trash_chunk();
 		if (!appctx->cli_ctx.cmdline) {
 			cli_err(appctx, "Failed to alloc a buffer to process the command line.\n");
-			applet_set_error(appctx);
-			b_reset(&appctx->inbuf);
-			goto end;
+			goto error;
 		}
 	}
 
@@ -995,10 +1045,26 @@ int cli_parse_cmdline(struct appctx *appctx)
 
 		if (!len) {
 			if (!b_room(buf) || (b_data(&appctx->inbuf) > b_room(buf) - 1)) {
-				cli_err(appctx, "The command line is too big for the buffer size. Please change tune.bufsize in the configuration to use a bigger command.\n");
-				applet_set_error(appctx);
-				b_reset(&appctx->inbuf);
-				goto end;
+				if (!(appctx->st1 & APPCTX_CLI_ST1_PAYLOAD)) {
+					/* The command line is too big and payload, if any, cannot be dynamic */
+					cli_err(appctx, "The command line is too big for the buffer size."
+						" Please change 'tune.bufsize' in the configuration to use a bigger command.\n");
+					goto error;
+				}
+
+				/* try to reallocate a bigger payload */
+				if (!appctx->cli_ctx.max_payload_sz ||
+				    b_data(&appctx->inbuf) + b_data(buf) + 1 > appctx->cli_ctx.max_payload_sz ||
+				    cli_try_realloc_payload(appctx, buf, 2 * (b_data(&appctx->inbuf) + b_data(buf))) == -1) {
+					/* Payload is too big or realloc failed */
+					cli_err(appctx, "The payload is too big."
+						" Please change 'tune.cli.max-payload-size' in the configuration to use a bigger payload.\n");
+					goto error;
+				}
+
+				/* A bigger payload buffer was allcated, wait for more data */
+				b_xfer(buf, &appctx->inbuf, b_data(&appctx->inbuf));
+				applet_fl_clr(appctx, APPCTX_FL_INBLK_FULL);
 			}
 			break;
 		}
@@ -1105,6 +1171,11 @@ int cli_parse_cmdline(struct appctx *appctx)
 	if (appctx->st0 != CLI_ST_PARSE_CMDLINE)
 		ret = 1;
 	return ret;
+
+ error:
+	applet_set_error(appctx);
+	b_reset(&appctx->inbuf);
+	goto end;
 }
 
 /* This I/O handler runs as an applet embedded in a stream connector. It is
@@ -1300,10 +1371,12 @@ void cli_io_handler(struct appctx *appctx)
 			if (appctx->st1 & APPCTX_CLI_ST1_LASTCMD) {
 				applet_reset_svcctx(appctx);
 				free_trash_chunk(appctx->cli_ctx.cmdline);
+				if (appctx->st1 & APPCTX_CLI_ST1_DYN_PAYLOAD)
+					free(b_orig(&appctx->cli_ctx.payload));
 				appctx->cli_ctx.payload_pat = NULL;
 				appctx->cli_ctx.cmdline = NULL;
 				appctx->cli_ctx.payload = BUF_NULL;
-				appctx->st1 &= ~APPCTX_CLI_ST1_LASTCMD;
+				appctx->st1 &= ~(APPCTX_CLI_ST1_LASTCMD|APPCTX_CLI_ST1_DYN_PAYLOAD);
 				if (appctx->st1 & APPCTX_CLI_ST1_INTER) {
 					appctx->st0 = CLI_ST_PARSE_CMDLINE;
 					applet_will_consume(appctx);
@@ -1347,6 +1420,10 @@ void cli_io_handler(struct appctx *appctx)
 static void cli_release_handler(struct appctx *appctx)
 {
 	free_trash_chunk(appctx->cli_ctx.cmdline);
+	if (appctx->st1 & APPCTX_CLI_ST1_DYN_PAYLOAD) {
+		free(b_orig(&appctx->cli_ctx.payload));
+		appctx->st1 &= ~APPCTX_CLI_ST1_DYN_PAYLOAD;
+	}
 
 	if (appctx->cli_ctx.io_release) {
 		EXEC_CTX_NO_RET(appctx->cli_ctx.kw->exec_ctx, appctx->cli_ctx.io_release(appctx));
@@ -4042,6 +4119,7 @@ INITCALL1(STG_REGISTER, cli_register_kw, &cli_kws);
 
 static struct cfg_kw_list cfg_kws = {ILH, {
 	{ CFG_GLOBAL, "stats", cli_parse_global },
+	{ CFG_GLOBAL, "tune.cli.max-payload-size", cli_parse_global_max_payload_size },
 	{ 0, NULL, NULL },
 }};
 

src/haproxy.c

@@ -201,6 +201,7 @@ struct global global = {
 #endif
 		.nb_stk_ctr = MAX_SESS_STKCTR,
 		.default_shards = -2, /* by-group */
+		.cli_max_payload_sz = 128 * 1024,
 	},
 #ifdef USE_OPENSSL
 #ifdef DEFAULT_MAXSSLCONN