mirrors/haproxy
1
0
Fork 0
mirror of https://git.haproxy.org/git/haproxy.git/ synced 2025-08-07 15:47:01 +02:00
Code Issues Projects Releases Wiki Activity

OPTIM: htx: inline the most common memcpy(8)

Browse Source 
On high traffic benchmarks, it's visible the the CPU is dominated by
calls to memcpy(), and many of those come from htx functions. It was
measured that 63% of those coming from htx are made on 8-byte blocks
which really are not worth a call to the function since a single
read-write cycle does it fine.

This commit adds an inline htx_memcpy() function that explicitly
checks for this length and just copies the data without that call.
It's even likely that it could be detected on const sizes, though
that was not done. This is already effective in reducing the number
of calls to memcpy().
This commit is contained in:
Willy Tarreau 2023-02-02 15:32:20 +01:00
parent 24d5b72ca9
commit 23aa79d9a9
1 changed files with 31 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
src/htx.c
View File

@ -12,9 +12,21 @@
#include <haproxy/chunk.h> #include <haproxy/chunk.h>
#include <haproxy/htx.h> #include <haproxy/htx.h>
#include <haproxy/net_helper.h>
struct htx htx_empty = { .size = 0, .data = 0, .head = -1, .tail = -1, .first = -1 }; struct htx htx_empty = { .size = 0, .data = 0, .head = -1, .tail = -1, .first = -1 };
/* tests show that 63% of these calls are for 64-bit chunks, so better avoid calling
* memcpy() for that!
*/
static inline __attribute__((always_inline)) void htx_memcpy(void *dst, void *src, size_t len)
{
if (likely(len == 8))
write_u64(dst, read_u64(src));
else
memcpy(dst, src, len);
}
/* Defragments an HTX message. It removes unused blocks and unwraps the payloads /* Defragments an HTX message. It removes unused blocks and unwraps the payloads
* part. A temporary buffer is used to do so. This function never fails. Most of * part. A temporary buffer is used to do so. This function never fails. Most of
* time, we need keep a ref on a specific HTX block. Thus is <blk> is set, the * time, we need keep a ref on a specific HTX block. Thus is <blk> is set, the
@ -50,7 +62,7 @@ struct htx_blk *htx_defrag(struct htx *htx, struct htx_blk *blk, uint32_t blkinf
continue; continue;
blksz = htx_get_blksz(oldblk); blksz = htx_get_blksz(oldblk);
memcpy((void *)tmp->blocks + addr, htx_get_blk_ptr(htx, oldblk), blksz); htx_memcpy((void *)tmp->blocks + addr, htx_get_blk_ptr(htx, oldblk), blksz);
/* update the start-line position */ /* update the start-line position */
if (htx->first == old) if (htx->first == old)
@ -80,7 +92,7 @@ struct htx_blk *htx_defrag(struct htx *htx, struct htx_blk *blk, uint32_t blkinf
htx->head_addr = htx->end_addr = 0; htx->head_addr = htx->end_addr = 0;
htx->tail_addr = addr; htx->tail_addr = addr;
htx->flags &= ~HTX_FL_FRAGMENTED; htx->flags &= ~HTX_FL_FRAGMENTED;
memcpy((void *)htx->blocks, (void *)tmp->blocks, htx->size); htx_memcpy((void *)htx->blocks, (void *)tmp->blocks, htx->size);
return ((blkpos == -1) ? NULL : htx_get_blk(htx, blkpos)); return ((blkpos == -1) ? NULL : htx_get_blk(htx, blkpos));
} }
@ -553,7 +565,7 @@ struct htx_blk *htx_add_data_atonce(struct htx *htx, struct ist data)
append_data: append_data:
/* Append data and update the block itself */ /* Append data and update the block itself */
ptr = htx_get_blk_ptr(htx, tailblk); ptr = htx_get_blk_ptr(htx, tailblk);
memcpy(ptr+sz, data.ptr, len); htx_memcpy(ptr+sz, data.ptr, len);
htx_change_blk_value_len(htx, tailblk, sz+len); htx_change_blk_value_len(htx, tailblk, sz+len);
if (data.len == len) { if (data.len == len) {
@ -568,7 +580,7 @@ struct htx_blk *htx_add_data_atonce(struct htx *htx, struct ist data)
return NULL; return NULL;
blk->info += data.len; blk->info += data.len;
memcpy(htx_get_blk_ptr(htx, blk), data.ptr, data.len); htx_memcpy(htx_get_blk_ptr(htx, blk), data.ptr, data.len);
end: end:
BUG_ON((int32_t)htx->tail_addr < 0); BUG_ON((int32_t)htx->tail_addr < 0);
@ -600,7 +612,7 @@ struct htx_blk *htx_replace_blk_value(struct htx *htx, struct htx_blk *blk,
if (ret == 1) { /* Replace in place */ if (ret == 1) { /* Replace in place */
if (delta <= 0) { if (delta <= 0) {
/* compression: copy new data first then move the end */ /* compression: copy new data first then move the end */
memcpy(old.ptr, new.ptr, new.len); htx_memcpy(old.ptr, new.ptr, new.len);
memmove(old.ptr + new.len, istend(old), memmove(old.ptr + new.len, istend(old),
istend(v) - istend(old)); istend(v) - istend(old));
} }
@ -608,7 +620,7 @@ struct htx_blk *htx_replace_blk_value(struct htx *htx, struct htx_blk *blk,
/* expansion: move the end first then copy new data */ /* expansion: move the end first then copy new data */
memmove(old.ptr + new.len, istend(old), memmove(old.ptr + new.len, istend(old),
istend(v) - istend(old)); istend(v) - istend(old));
memcpy(old.ptr, new.ptr, new.len); htx_memcpy(old.ptr, new.ptr, new.len);
} }
/* set the new block size and update HTX message */ /* set the new block size and update HTX message */
@ -619,19 +631,19 @@ struct htx_blk *htx_replace_blk_value(struct htx *htx, struct htx_blk *blk,
void *ptr = htx_get_blk_ptr(htx, blk); void *ptr = htx_get_blk_ptr(htx, blk);
/* Copy the name, if any */ /* Copy the name, if any */
memcpy(ptr, n.ptr, n.len); htx_memcpy(ptr, n.ptr, n.len);
ptr += n.len; ptr += n.len;
/* Copy value before old part, if any */ /* Copy value before old part, if any */
memcpy(ptr, v.ptr, old.ptr - v.ptr); htx_memcpy(ptr, v.ptr, old.ptr - v.ptr);
ptr += old.ptr - v.ptr; ptr += old.ptr - v.ptr;
/* Copy new value */ /* Copy new value */
memcpy(ptr, new.ptr, new.len); htx_memcpy(ptr, new.ptr, new.len);
ptr += new.len; ptr += new.len;
/* Copy value after old part, if any */ /* Copy value after old part, if any */
memcpy(ptr, istend(old), istend(v) - istend(old)); htx_memcpy(ptr, istend(old), istend(v) - istend(old));
/* set the new block size and update HTX message */ /* set the new block size and update HTX message */
htx_set_blk_value_len(blk, v.len + delta); htx_set_blk_value_len(blk, v.len + delta);
@ -658,7 +670,7 @@ struct htx_blk *htx_replace_blk_value(struct htx *htx, struct htx_blk *blk,
*/ */
memmove(old.ptr + offset + new.len, old.ptr + offset + old.len, memmove(old.ptr + offset + new.len, old.ptr + offset + old.len,
istend(v) - istend(old)); istend(v) - istend(old));
memcpy(old.ptr + offset, new.ptr, new.len); htx_memcpy(old.ptr + offset, new.ptr, new.len);
} }
return blk; return blk;
} }
@ -708,7 +720,7 @@ struct htx_ret htx_xfer_blks(struct htx *dst, struct htx *src, uint32_t count,
if (!dstblk) if (!dstblk)
break; break;
dstblk->info = info; dstblk->info = info;
memcpy(htx_get_blk_ptr(dst, dstblk), htx_get_blk_ptr(src, blk), sz); htx_memcpy(htx_get_blk_ptr(dst, dstblk), htx_get_blk_ptr(src, blk), sz);
count -= sizeof(dstblk) + sz; count -= sizeof(dstblk) + sz;
if (blk->info != info) { if (blk->info != info) {
@ -807,7 +819,7 @@ struct htx_blk *htx_replace_header(struct htx *htx, struct htx_blk *blk,
/* Finally, copy data. */ /* Finally, copy data. */
ptr = htx_get_blk_ptr(htx, blk); ptr = htx_get_blk_ptr(htx, blk);
ist2bin_lc(ptr, name); ist2bin_lc(ptr, name);
memcpy(ptr + name.len, value.ptr, value.len); htx_memcpy(ptr + name.len, value.ptr, value.len);
return blk; return blk;
} }
@ -860,9 +872,9 @@ struct htx_sl *htx_replace_stline(struct htx *htx, struct htx_blk *blk, const st
HTX_SL_P2_LEN(sl) = p2.len; HTX_SL_P2_LEN(sl) = p2.len;
HTX_SL_P3_LEN(sl) = p3.len; HTX_SL_P3_LEN(sl) = p3.len;
memcpy(HTX_SL_P1_PTR(sl), p1.ptr, p1.len); htx_memcpy(HTX_SL_P1_PTR(sl), p1.ptr, p1.len);
memcpy(HTX_SL_P2_PTR(sl), p2.ptr, p2.len); htx_memcpy(HTX_SL_P2_PTR(sl), p2.ptr, p2.len);
memcpy(HTX_SL_P3_PTR(sl), p3.ptr, p3.len); htx_memcpy(HTX_SL_P3_PTR(sl), p3.ptr, p3.len);
return sl; return sl;
} }
@ -982,7 +994,7 @@ size_t htx_add_data(struct htx *htx, const struct ist data)
append_data: append_data:
/* Append data and update the block itself */ /* Append data and update the block itself */
ptr = htx_get_blk_ptr(htx, tailblk); ptr = htx_get_blk_ptr(htx, tailblk);
memcpy(ptr + sz, data.ptr, len); htx_memcpy(ptr + sz, data.ptr, len);
htx_change_blk_value_len(htx, tailblk, sz+len); htx_change_blk_value_len(htx, tailblk, sz+len);
BUG_ON((int32_t)htx->tail_addr < 0); BUG_ON((int32_t)htx->tail_addr < 0);
@ -997,7 +1009,7 @@ size_t htx_add_data(struct htx *htx, const struct ist data)
return 0; return 0;
blk->info += len; blk->info += len;
memcpy(htx_get_blk_ptr(htx, blk), data.ptr, len); htx_memcpy(htx_get_blk_ptr(htx, blk), data.ptr, len);
return len; return len;
} }
@ -1076,7 +1088,7 @@ int htx_append_msg(struct htx *dst, const struct htx *src)
if (!newblk) if (!newblk)
goto error; goto error;
newblk->info = blk->info; newblk->info = blk->info;
memcpy(htx_get_blk_ptr(dst, newblk), htx_get_blk_ptr(src, blk), blksz); htx_memcpy(htx_get_blk_ptr(dst, newblk), htx_get_blk_ptr(src, blk), blksz);
} }
return 1; return 1;