MINOR: stats-file: introduce shm-stats-file directive

add initial support for the "shm-stats-file" directive and
associated "shm-stats-file-max-objects" directive. For now they are
flagged as experimental directives.

The shared memory file is automatically created by the first process.
The file is created using open() so it is up to the user to provide
relevant path (either on regular filesystem or ramfs for performance
reasons). The directive takes only one argument which is path of the
shared memory file. It is passed as-is to open().

The maximum number of objects per thread-group (hard limit) that can be
stored in the shm is defined by "shm-stats-file-max-objects" directive,

Upon initial creation, the main shm stats file header is provisioned with
the version which must remains the same to be compatible between processes
and defaults to 2k. which means approximately 1mb max per thread group
and should cover most setups. When the limit is reached (during startup)
an error is reported by haproxy which invites the user to increase the
"shm-stats-file-max-objects" if desired, but this means more memory will
be allocated. Actual memory usage is low at start, because only the mmap
(mapping) is provisionned with the maximum number of objects to avoid
relocating the memory area during runtime, but the actual shared memory
file is dynamically resized when objects are added (resized by following
half power of 2 curve when new objects are added, see upcoming commits)

For now only the file is created, further logic will be implemented in
upcoming commits.

include/haproxy/defaults.h

@@ -366,6 +366,13 @@
 #define STATS_VERSION_STRING " version " HAPROXY_VERSION ", released " HAPROXY_DATE
 #endif
 
+/* specifies the default max number of object per thread group that the shm stats file
+ * will be able to handle
+ */
+#ifndef SHM_STATS_FILE_MAX_OBJECTS
+#define SHM_STATS_FILE_MAX_OBJECTS 2000
+#endif
+
 /* This is the default statistics URI */
 #ifdef CONFIG_STATS_DEFAULT_URI
 #define STATS_DEFAULT_URI CONFIG_STATS_DEFAULT_URI

include/haproxy/global-t.h

@@ -167,6 +167,7 @@ struct global {
 	char *server_state_base;   /* path to a directory where server state files can be found */
 	char *server_state_file;   /* path to the file where server states are loaded from */
 	char *stats_file;          /* path to stats-file */
+	char *shm_stats_file;      /* path to shm-stats-file */
 	unsigned char cluster_secret[16]; /* 128 bits of an SHA1 digest of a secret defined as ASCII string */
 	struct {
 		int maxpollevents; /* max number of poll events at once */

include/haproxy/stats-file-t.h

@@ -1,6 +1,11 @@
 #ifndef _HAPROXY_STATS_FILE_T_H
 #define _HAPROXY_STATS_FILE_T_H
 
+#include <inttypes.h>
+#include <haproxy/compat.h>
+#include <haproxy/counters-t.h>
+#include <haproxy/guid-t.h>
+
 /* Sections present in stats-file separated by header lines. */
 enum stfile_domain {
 	STFILE_DOMAIN_UNSET = 0,
@@ -9,4 +14,21 @@ enum stfile_domain {
 	STFILE_DOMAIN_PX_BE,  /* #be headers */
 };
 
+#define SHM_STATS_FILE_VER_MAJOR         1
+#define SHM_STATS_FILE_VER_MINOR         0
+
+/* header for shm stats file ("shm-stats-file") */
+struct shm_stats_file_hdr {
+	/* to check if the header is compatible with current haproxy version */
+	struct {
+		uint8_t major;
+		uint8_t minor;
+	} version;
+};
+
+struct shm_stats_file_object {
+};
+
+#define SHM_STATS_FILE_MAPPING_SIZE(obj) (sizeof(struct shm_stats_file_hdr) + (obj) * sizeof(struct shm_stats_file_object))
+
 #endif /* _HAPROXY_STATS_FILE_T_H */

include/haproxy/stats-file.h

@@ -8,6 +8,9 @@
 #include <haproxy/buf-t.h>
 #include <haproxy/stats-t.h>
 
+extern struct shm_stats_file_hdr *shm_stats_file_hdr;
+extern int shm_stats_file_max_objects;
+
 int stats_dump_fields_file(struct buffer *out,
                            const struct field *stats, size_t stats_count,
                            struct show_stat_ctx *ctx);
@@ -21,5 +24,6 @@ void stats_dump_file_header(int type, struct buffer *out);
 #define STAT_FILE_MAX_COL_COUNT    (ST_I_PX_MAX*2)
 
 void apply_stats_file(void);
+int shm_stats_file_prepare(void);
 
 #endif /* _HAPROXY_STATS_FILE_H */

src/cfgparse-global.c

@@ -24,6 +24,7 @@
 #include <haproxy/log.h>
 #include <haproxy/peers.h>
 #include <haproxy/protocol.h>
+#include <haproxy/stats-file.h>
 #include <haproxy/stress.h>
 #include <haproxy/tools.h>
 
@@ -1599,6 +1600,52 @@ static int cfg_parse_global_env_opts(char **args, int section_type,
 	return 0;
 }
 
+static int cfg_parse_global_shm_stats_file(char **args, int section_type,
+				           struct proxy *curpx, const struct proxy *defpx,
+				           const char *file, int line, char **err)
+{
+	if (!experimental_directives_allowed) {
+		memprintf(err, "'%s' directive is experimental, must be allowed via a global 'expose-experimental-directives'", args[0]);
+		return -1;
+	}
+
+	if (global.shm_stats_file != NULL) {
+		memprintf(err, "'%s' already specified.\n", args[0]);
+		return -1;
+	}
+
+	if (!*(args[1])) {
+		memprintf(err, "'%s' expect one argument: a file path.\n", args[0]);
+		return -1;
+	}
+
+	global.shm_stats_file = strdup(args[1]);
+	return 0;
+}
+
+static int cfg_parse_global_shm_stats_file_max_objects(char **args, int section_type,
+				                       struct proxy *curpx, const struct proxy *defpx,
+				                       const char *file, int line, char **err)
+{
+	if (!experimental_directives_allowed) {
+		memprintf(err, "'%s' directive is experimental, must be allowed via a global 'expose-experimental-directives'", args[0]);
+		return -1;
+	}
+
+	if (shm_stats_file_max_objects != -1) {
+		memprintf(err, "'%s' already specified.\n", args[0]);
+		return -1;
+	}
+
+	if (!*(args[1])) {
+		memprintf(err, "'%s' expect one argument: max objects number.\n", args[0]);
+		return -1;
+	}
+
+	shm_stats_file_max_objects = atoi(args[1]);
+	return 0;
+}
+
 static int cfg_parse_global_parser_pause(char **args, int section_type,
                                          struct proxy *curpx, const struct proxy *defpx,
                                          const char *file, int line, char **err)
@@ -1809,6 +1856,8 @@ static struct cfg_kw_list cfg_kws = {ILH, {
 	{ CFG_GLOBAL, "quiet", cfg_parse_global_mode, KWF_DISCOVERY },
 	{ CFG_GLOBAL, "resetenv", cfg_parse_global_env_opts, KWF_DISCOVERY },
 	{ CFG_GLOBAL, "setenv", cfg_parse_global_env_opts, KWF_DISCOVERY },
+	{ CFG_GLOBAL, "shm-stats-file", cfg_parse_global_shm_stats_file },
+	{ CFG_GLOBAL, "shm-stats-file-max-objects", cfg_parse_global_shm_stats_file_max_objects },
 	{ CFG_GLOBAL, "stress-level", cfg_parse_global_stress_level },
 	{ CFG_GLOBAL, "tune.bufsize", cfg_parse_global_tune_opts },
 	{ CFG_GLOBAL, "tune.chksize", cfg_parse_global_unsupported_opts },

src/haproxy.c

@@ -2105,6 +2105,13 @@ static void step_init_2(int argc, char** argv)
 		exit(1);
 	}
 
+	/* now that config was parsed and checked
+	 * prepare and preload shm-stats-file (if set)
+	 */
+	err_code |= shm_stats_file_prepare();
+	if (err_code & (ERR_ABORT|ERR_FATAL))
+		exit(1);
+
 	/* update the ready date to also account for the check time */
 	clock_update_date(0, 1);
 	clock_adjust_now_offset();

src/stats-file.c

@@ -1,13 +1,18 @@
 #include <haproxy/stats-file.h>
 
+#include <errno.h>
+#include <fcntl.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
 
 #include <import/ebmbtree.h>
 #include <import/ebsttree.h>
 #include <import/ist.h>
 #include <haproxy/api.h>
+#include <haproxy/atomic.h>
 #include <haproxy/buf.h>
 #include <haproxy/chunk.h>
 #include <haproxy/clock.h>
@@ -21,7 +26,13 @@
 #include <haproxy/proxy-t.h>
 #include <haproxy/server-t.h>
 #include <haproxy/stats.h>
+#include <haproxy/task.h>
 #include <haproxy/time.h>
+#include <haproxy/tools.h>
+
+struct shm_stats_file_hdr *shm_stats_file_hdr = NULL;
+static int shm_stats_file_fd = -1;
+int shm_stats_file_max_objects = -1;
 
 /* Dump all fields from <stats> into <out> for stats-file. */
 int stats_dump_fields_file(struct buffer *out,
@@ -441,3 +452,109 @@ void apply_stats_file(void)
 	ha_free(&line);
 	fclose(file);
 }
+
+/* returns 1 if <hdr> shm version is compatible with current version
+ * defined in stats-file-t.h or 0 if it is not compatible.
+ */
+static int shm_stats_file_check_ver(struct shm_stats_file_hdr *hdr)
+{
+	/* for now we don't even support minor version difference but this may
+	 * change later
+	 */
+	if (hdr->version.major != SHM_STATS_FILE_VER_MAJOR ||
+	    hdr->version.minor != SHM_STATS_FILE_VER_MINOR)
+		return 0;
+	return 1;
+}
+
+/* since shm file was opened using O_APPEND flag, let's grow
+ * the file by <bytes> in an atomic manner (O_APPEND offers such guarantee),
+ * so that even if multiple processes try to grow the file simultaneously,
+ * the file can only grow bigger and never shrink
+ *
+ * We do this way because ftruncate() between multiple processes
+ * could result in the file being shrunk if one of the process
+ * is not aware that the file was already expanded in the meantime
+ *
+ * Returns 1 on success and 0 on failure
+ */
+static int shm_file_grow(unsigned int bytes)
+{
+	char buf[1024] = {0};
+	ssize_t ret;
+
+	while (bytes) {
+		ret = write(shm_stats_file_fd, buf, MIN(sizeof(buf), bytes));
+		if (ret <= 0)
+			return 0;
+		bytes -= ret;
+	}
+	return 1;
+}
+
+/* prepare and and initialize shm stats memory file as needed */
+int shm_stats_file_prepare(void)
+{
+	int first = 0; // process responsible for initializing the shm memory
+
+	/* do nothing if master process or shm_stats_file not configured */
+	if (master || !global.shm_stats_file)
+		return ERR_NONE;
+
+	/* compute final shm_stats_file_max_objects value */
+	if (shm_stats_file_max_objects == -1)
+		shm_stats_file_max_objects = SHM_STATS_FILE_MAX_OBJECTS * global.nbtgroups;
+	else
+		shm_stats_file_max_objects = shm_stats_file_max_objects * global.nbtgroups;
+
+	shm_stats_file_fd = open(global.shm_stats_file, O_RDWR | O_APPEND | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR);
+	if (shm_stats_file_fd == -1) {
+		shm_stats_file_fd = open(global.shm_stats_file, O_RDWR | O_APPEND, S_IRUSR | S_IWUSR);
+		if (shm_stats_file_fd == -1) {
+			ha_alert("config: cannot open shm stats file '%s': %s\n", global.shm_stats_file, strerror(errno));
+			return ERR_ALERT | ERR_FATAL;
+		}
+	}
+	else {
+		first = 1;
+		if (shm_file_grow(sizeof(*shm_stats_file_hdr)) == 0) {
+			ha_alert("config: unable to resize shm stats file '%s'\n", global.shm_stats_file);
+			return ERR_ALERT | ERR_FATAL;
+		}
+	}
+	/* mmap maximum contiguous address space for expected objects even if the backing shm is
+	 * smaller: it will allow for on the fly shm resizing without having to remap
+	 */
+	shm_stats_file_hdr = mmap(NULL,
+	                          SHM_STATS_FILE_MAPPING_SIZE(shm_stats_file_max_objects),
+	                          PROT_READ | PROT_WRITE, MAP_SHARED, shm_stats_file_fd, 0);
+	if (shm_stats_file_hdr == MAP_FAILED || shm_stats_file_hdr == NULL) {
+		ha_alert("config: failed to map shm stats file '%s'\n", global.shm_stats_file);
+		return ERR_ALERT | ERR_FATAL;
+	}
+
+	if (first) {
+		/* let's init some members */
+		memset(shm_stats_file_hdr, 0, sizeof(*shm_stats_file_hdr));
+		shm_stats_file_hdr->version.major = SHM_STATS_FILE_VER_MAJOR;
+		shm_stats_file_hdr->version.minor = SHM_STATS_FILE_VER_MINOR;
+	}
+	else if (!shm_stats_file_check_ver(shm_stats_file_hdr))
+		goto err_version;
+
+ end:
+	return ERR_NONE;
+
+ err_version:
+		ha_warning("config: incompatible map shm stats file version '%s'\n", global.shm_stats_file);
+	return ERR_WARN;
+}
+
+static void cleanup_shm_stats_file(void)
+{
+	if (shm_stats_file_hdr) {
+		munmap(shm_stats_file_hdr, SHM_STATS_FILE_MAPPING_SIZE(shm_stats_file_max_objects));
+		close(shm_stats_file_fd);
+	}
+}
+REGISTER_POST_DEINIT(cleanup_shm_stats_file);