MEDIUM: dynbuf: refrain from offering a buffer if more critical ones are waiting

Now b_alloc() will check the queues at the same and higher criticality
levels before allocating a buffer, and will refrain from allocating one
if these are not empty. The purpose is to put some priorities in the
allocation order so that most critical allocators are offered a chance
to complete.

However in order to permit a freshly dequeued task to allocate again while
siblings are still in the queue, there is a special DB_F_NOQUEUE flag to
pass to b_alloc() that will take care of this special situation.

include/haproxy/dynbuf-t.h

@@ -50,7 +50,8 @@
  *     buffers). If these fail, we can't boot.
  *
  * Please DO NOT CHANGE THESE LEVELS without first getting a full understanding
- * of how all this works and touching the DB_CRIT_TO_QUEUE() macro below!
+ * of how all this works and touching the DB_F_CRIT_MASK and DB_CRIT_TO_QUEUE()
+ * macros below!
  */
 enum dynbuf_crit {
 	DB_GROW_RING = 0, // used to grow an existing buffer ring
@@ -64,6 +65,14 @@ enum dynbuf_crit {
 	DB_PERMANENT,     // buffers permanently allocated.
 };
 
+/* The values above are expected to be passed to b_alloc(). In addition, some
+ * Extra flags can be passed by oring the crit value above with one of these
+ * high-bit flags.
+ */
+#define DB_F_NOQUEUE   0x80000000U   // ignore presence of others in queue
+#define DB_F_CRIT_MASK 0x000000FFU   // mask to keep the criticality bits
+
+
 /* We'll deal with 4 queues, with indexes numbered from 0 to 3 based on the
  * criticality of the allocation. All criticality levels are mapped to a 2-bit
  * queue index. While some levels never use the queue (the first two), some of

include/haproxy/dynbuf.h

@@ -56,21 +56,37 @@ static inline int buffer_almost_full(const struct buffer *buf)
 /* Functions below are used for buffer allocation */
 /**************************************************/
 
+/* returns non-zero if one may try to allocate a buffer for criticality flags
+ * <crit> (made of a criticality and optional flags).
+ */
+static inline int b_may_alloc_for_crit(uint crit)
+{
+	int q = DB_CRIT_TO_QUEUE(crit & DB_F_CRIT_MASK);
+
+	/* if this queue or any more critical ones have entries, we must wait */
+	if (!(crit & DB_F_NOQUEUE) && th_ctx->bufq_map & ((2 << q) - 1))
+		return 0;
+
+	return 1;
+}
+
 /* Ensures that <buf> is allocated, or allocates it. If no memory is available,
  * ((char *)1) is assigned instead with a zero size. The allocated buffer is
  * returned, or NULL in case no memory is available. Since buffers only contain
  * user data, poisonning is always disabled as it brings no benefit and impacts
  * performance. Due to the difficult buffer_wait management, they are not
- * subject to forced allocation failures either.
+ * subject to forced allocation failures either. If other waiters are present
+ * at higher criticality levels, we refrain from allocating.
  */
 #define b_alloc(_buf, _crit)						\
 ({									\
-	char *_area;				\
+	char *_area = NULL;						\
 	struct buffer *_retbuf = _buf;					\
-	enum dynbuf_crit _criticality __maybe_unused = _crit;	\
+	uint _criticality = _crit;					\
 									\
 	if (!_retbuf->size) {						\
 		*_retbuf = BUF_WANTED;					\
+		if (b_may_alloc_for_crit(_criticality))			\
 			_area = pool_alloc_flag(pool_head_buffer, POOL_F_NO_POISON | POOL_F_NO_FAIL); \
 		if (unlikely(!_area)) {					\
 			activity[tid].buf_wait++;			\