MEDIUM: log: Unique ID

The Unique ID, is an ID generated with several informations. You can use
a log-format string to customize it, with the "unique-id-format" keyword,
and insert it in the request header, with the "unique-id-header" keyword.

doc/configuration.txt

@@ -1117,6 +1117,8 @@ timeout server                            X          -         X         X
 timeout srvtimeout          (deprecated)  X          -         X         X
 timeout tarpit                            X          X         X         X
 transparent                 (deprecated)  X          -         X         X
+unique-id-format                          X          X         X         -
+unique-id-header                          X          X         X         -
 use_backend                               -          X         X         -
 use-server                                -          -         X         X
 ------------------------------------+----------+----------+---------+---------
@@ -6564,6 +6566,60 @@ transparent (deprecated)
 
   See also: "option transparent"
 
+unique-id-format <string>
+  Generate a unique ID for each request.
+  May be used in sections :   defaults | frontend | listen | backend
+                                  yes  |    yes   |   yes  |   no
+  Arguments :
+    <string>   is a log-format string.
+
+    This keyword creates a ID for each request using the custom log format. A
+    unique ID is useful to trace a request passing through many components of
+    a complex infrastructure. The newly created ID may also be logged using the
+    %ID tag the log-format string.
+
+    The format should be composed from elements that are guaranteed to be
+    unique when combined together. For instance, if multiple haproxy instances
+    are involved, it might be important to include the node name. It is often
+    needed to log the incoming connection's source and destination addresses
+    and ports. Note that since multiple requests may be performed over the same
+    connection, including a request counter may help differentiate them.
+    Similarly, a timestamp may protect against a rollover of the counter.
+    Logging the process ID will avoid collisions after a service restart.
+
+    It is recommended to use hexadecimal notation for many fields since it
+    makes them more compact and saves space in logs.
+
+    Example:
+
+        unique-id-format %{+X}o\ %Ci:%Cp_%Fi:%Fp_%Ts_%rt:%pid
+
+        will generate:
+
+               7F000001:8296_7F00001E:1F90_4F7B0A69_0003:790A
+
+  See also: "unique-id-header"
+
+unique-id-header <name>
+  Add a unique ID header in the HTTP request.
+  May be used in sections :   defaults | frontend | listen | backend
+                                  yes  |    yes   |   yes  |   no
+  Arguments :
+    <name>   is the name of the header.
+
+    Add a unique-id header in the HTTP request sent to the server, using the
+    unique-id-format. It can't work if the unique-id-format doesn't exist.
+
+    Example:
+
+        unique-id-format %{+X}o\ %Ci:%Cp_%Fi:%Fp_%Ts_%rt:%pid
+        unique-id-header X-Unique-ID
+
+        will generate:
+
+           X-Unique-ID: 7F000001:8296_7F00001E:1F90_4F7B0A69_0003:790A
+
+    See also: "unique-id-format"
 
 use_backend <backend> if <condition>
 use_backend <backend> unless <condition>
@@ -8919,6 +8975,7 @@ Please refer to the table below for currently defined variables :
   |   | %Fi  | frontend_ip                                   | IP          |
   |   | %Fp  | frontend_port                                 | numeric     |
   |   | %H   | hostname                                      | string      |
+  |   | %ID  | unique-id                                     | string      |
   |   | %Si  | server_IP                                     | IP          |
   |   | %Sp  | server_port                                   | numeric     |
   |   | %T   | gmt_date_time                                 | date        |

include/proto/log.h

@@ -33,6 +33,7 @@
 #include <types/session.h>
 
 extern struct pool_head *pool2_requri;
+extern struct pool_head *pool2_uniqueid;
 
 extern char *log_format;
 extern char default_tcp_log_format[];

include/types/log.h

@@ -31,6 +31,8 @@
 #define NB_LOG_FACILITIES       24
 #define NB_LOG_LEVELS           8
 #define SYSLOG_PORT             514
+#define UNIQUEID_LEN            128
+
 
 /* lists of fields that can be logged */
 enum {
@@ -86,6 +88,7 @@ enum {
 	LOG_FMT_HDRRESPONSLIST,
 	LOG_FMT_REQ,
 	LOG_FMT_HOSTNAME,
+	LOG_FMT_UNIQUEID,
 };
 
 /* enum for parse_logformat */

include/types/proxy.h

@@ -290,6 +290,8 @@ struct proxy {
 	struct proxy *next;
 	struct list logsrvs;
 	struct list logformat; 			/* log_format linked list */
+	char *header_unique_id; 		/* unique-id header */
+	struct list format_unique_id;		/* unique-id format */
 	int to_log;				/* things to be logged (LW_*) */
 	int stop_time;                          /* date to stop listening, when stopping != 0 (int ticks) */
 	struct hdr_exp *req_exp;		/* regular expressions for request headers */

include/types/session.h

@@ -207,6 +207,7 @@ struct session {
 	void (*srv_error)(struct session *s,	/* the function to call upon unrecoverable server errors (or NULL) */
 			  struct stream_interface *si);
 	unsigned int uniq_id;			/* unique ID used for the traces */
+	char *unique_id;			/* custom unique ID */
 };
 
 /* parameters to configure tracked counters */

src/cfgparse.c

@@ -1550,6 +1550,18 @@ int cfg_parse_listen(const char *file, int linenum, char **args, int kwm)
 			LIST_ADDQ(&curproxy->logformat, &node->list);
 		}
 
+		/* copy default unique_id to curproxy */
+		list_for_each_entry(tmplf, &defproxy.format_unique_id, list) {
+			struct logformat_node *node = malloc(sizeof(struct logformat_node));
+			memcpy(node, tmplf, sizeof(struct logformat_node));
+			LIST_INIT(&node->list);
+			LIST_ADDQ(&curproxy->format_unique_id, &node->list);
+		}
+
+		/* copy default header unique id */
+		if (defproxy.header_unique_id)
+			curproxy->header_unique_id = strdup(defproxy.header_unique_id);
+
 		curproxy->grace  = defproxy.grace;
 		curproxy->conf.used_listener_id = EB_ROOT;
 		curproxy->conf.used_server_id = EB_ROOT;
@@ -4592,6 +4604,26 @@ int cfg_parse_listen(const char *file, int linenum, char **args, int kwm)
 			newsrv->prev_state = newsrv->state;
 		}
 	}
+
+	else if (strcmp(args[0], "unique-id-format") == 0) {
+		if (!*(args[1])) {
+			Alert("parsing [%s:%d] : %s expects an argument.\n", file, linenum, args[0]);
+			err_code |= ERR_ALERT | ERR_FATAL;
+			goto out;
+		}
+		parse_logformat_string(args[1], curproxy, &curproxy->format_unique_id, PR_MODE_HTTP);
+	}
+
+	else if (strcmp(args[0], "unique-id-header") == 0) {
+		if (!*(args[1])) {
+			Alert("parsing [%s:%d] : %s expects an argument.\n", file, linenum, args[0]);
+			err_code |= ERR_ALERT | ERR_FATAL;
+			goto out;
+		}
+		free(curproxy->header_unique_id);
+		curproxy->header_unique_id = strdup(args[1]);
+	}
+
 	else if (strcmp(args[0], "log-format") == 0) {
 		if (!*(args[1])) {
 			Alert("parsing [%s:%d] : %s expects an argument.\n", file, linenum, args[0]);

src/log.c

@@ -107,6 +107,7 @@ static const struct logformat_type logformat_keywords[] = {
 	{ "pid", LOG_FMT_PID, PR_MODE_TCP, NULL }, /* log pid */
 	{ "rt", LOG_FMT_COUNTER, PR_MODE_TCP, NULL }, /* log counter */
 	{ "H", LOG_FMT_HOSTNAME, PR_MODE_TCP, NULL }, /* Hostname */
+	{ "ID", LOG_FMT_UNIQUEID, PR_MODE_HTTP, NULL }, /* Unique ID */
 	{ 0, 0, 0, NULL }
 };
 
@@ -1305,6 +1306,16 @@ int build_logline(struct session *s, char *dst, size_t maxsize, struct list *lis
 					last_isspace = 0;
 				}
 				break;
+
+			case LOG_FMT_UNIQUEID: // %ID
+				src = s->unique_id;
+				ret = lf_text(tmplog, src, maxsize - (tmplog - dst), tmp);
+				if (ret == NULL)
+					goto out;
+				tmplog = ret;
+				last_isspace = 0;
+				break;
+
 		}
 	}
 

src/proto_http.c

@@ -268,6 +268,7 @@ void init_proto_http()
 	/* memory allocations */
 	pool2_requri = create_pool("requri", REQURI_LEN, MEM_F_SHARED);
 	pool2_capture = create_pool("capture", CAPTURE_LEN, MEM_F_SHARED);
+	pool2_uniqueid = create_pool("uniqueid", UNIQUEID_LEN, MEM_F_SHARED);
 }
 
 /*
@@ -858,6 +859,7 @@ extern const char sess_fin_state[8];
 extern const char *monthname[12];
 struct pool_head *pool2_requri;
 struct pool_head *pool2_capture;
+struct pool_head *pool2_uniqueid;
 
 /*
  * Capture headers from message starting at <som> according to header list
@@ -2397,6 +2399,10 @@ int http_wait_for_request(struct session *s, struct buffer *req, int an_bit)
 		}
 	}
 
+		if (!LIST_ISEMPTY(&s->fe->format_unique_id)) {
+			s->unique_id = pool_alloc2(pool2_uniqueid);
+		}
+
 	/* 4. We may have to convert HTTP/0.9 requests to HTTP/1.0 */
 	if (unlikely(msg->sl.rq.v_l == 0) && !http_upgrade_v09_to_v10(req, msg, txn))
 		goto return_bad_req;
@@ -3274,6 +3280,19 @@ int http_process_request(struct session *s, struct buffer *req, int an_bit)
 		get_srv_from_appsession(s, msg->sol + msg->sl.rq.u, msg->sl.rq.u_l);
 	}
 
+	/* add unique-id if "header-unique-id" is specified */
+
+	if (!LIST_ISEMPTY(&s->fe->format_unique_id))
+		build_logline(s, s->unique_id, UNIQUEID_LEN, &s->fe->format_unique_id);
+
+	if (s->fe->header_unique_id && s->unique_id) {
+		int ret = snprintf(trash, global.tune.bufsize, "%s: %s", s->fe->header_unique_id, s->unique_id);
+		if (ret < 0 || ret > global.tune.bufsize)
+			goto return_bad_req;
+		if(unlikely(http_header_add_tail(req, &txn->req, &txn->hdr_idx, trash) < 0))
+		   goto return_bad_req;
+	}
+
 	/*
 	 * 9: add X-Forwarded-For if either the frontend or the backend
 	 * asks for it.
@@ -7381,7 +7400,9 @@ void http_end_txn(struct session *s)
 	pool_free2(pool2_capture, txn->cli_cookie);
 	pool_free2(pool2_capture, txn->srv_cookie);
 	pool_free2(apools.sessid, txn->sessid);
+	pool_free2(pool2_uniqueid, s->unique_id);
 
+	s->unique_id = NULL;
 	txn->sessid = NULL;
 	txn->uri = NULL;
 	txn->srv_cookie = NULL;

src/proxy.c

@@ -439,6 +439,7 @@ void init_new_proxy(struct proxy *p)
 	LIST_INIT(&p->listener_queue);
 	LIST_INIT(&p->logsrvs);
 	LIST_INIT(&p->logformat);
+	LIST_INIT(&p->format_unique_id);
 
 	/* Timeouts are defined as -1 */
 	proxy_reset_timeouts(p);