From eaba76b02dd41e0a1a2e85a3e71f91dfc529916d Mon Sep 17 00:00:00 2001
From: Willy Tarreau <w@1wt.eu>
Date: Wed, 22 Mar 2023 14:55:25 +0100
Subject: [PATCH] MINOR: pools: intercept malloc_trim() instead of trying to
 plug holes

As reported by Miroslav in commit d8a97d8f6 ("BUG/MINOR: illegal use of
the malloc_trim() function if jemalloc is used") there are still occasional
cases where it's discovered that malloc_trim() is being used without its
suitability being checked first. This is a problem when using another
incompatible allocator. But there's a class of use cases we'll never be
able to cover, it's dynamic libraries loaded from Lua. In order to address
this more reliably, we now define our own malloc_trim() that calls the
previous one after checking that the feature is supported and that the
allocator is the expected one. This way child libraries that would call
it will also be safe.

The function is intentionally left defined all the time so that it will
be possible to clean up some code that uses it by removing ifdefs.
---
 include/haproxy/pool.h |  2 ++
 src/pool.c             | 23 ++++++++++++++++++++++-
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/include/haproxy/pool.h b/include/haproxy/pool.h
index a12990d24..d88b23fae 100644
--- a/include/haproxy/pool.h
+++ b/include/haproxy/pool.h
@@ -102,6 +102,8 @@ extern int mem_poison_byte;
 extern uint pool_debugging;
 
 int is_trim_enabled(void);
+int malloc_trim(size_t pad);
+
 void *pool_get_from_os(struct pool_head *pool);
 void pool_put_to_os(struct pool_head *pool, void *ptr);
 void *pool_alloc_nocache(struct pool_head *pool);
diff --git a/src/pool.c b/src/pool.c
index 908b3340f..aff858125 100644
--- a/src/pool.c
+++ b/src/pool.c
@@ -108,6 +108,7 @@ static int mem_fail_rate __read_mostly = 0;
 static int using_default_allocator __read_mostly = 1;
 static int disable_trim __read_mostly = 0;
 static int(*my_mallctl)(const char *, void *, size_t *, void *, size_t) = NULL;
+static int(*_malloc_trim)(size_t) = NULL;
 
 /* ask the allocator to trim memory pools.
  * This must run under thread isolation so that competing threads trying to
@@ -209,6 +210,9 @@ static void detect_allocator(void)
 		using_default_allocator = (malloc_default_zone() != NULL);
 #endif
 	}
+
+	/* detect presence of malloc_trim() */
+	_malloc_trim = get_sym_next_addr("malloc_trim");
 }
 
 int is_trim_enabled(void)
@@ -216,6 +220,23 @@ int is_trim_enabled(void)
 	return !disable_trim && using_default_allocator;
 }
 
+/* replace the libc's malloc_trim() so that we can also intercept the calls
+ * from child libraries when the allocator is not the default one.
+ */
+int malloc_trim(size_t pad)
+{
+	int ret = 0;
+
+	if (disable_trim)
+		return ret;
+
+	if (_malloc_trim && using_default_allocator) {
+		/* we're typically on glibc and not overridden */
+		ret = _malloc_trim(pad);
+	}
+	return ret;
+}
+
 static int mem_should_fail(const struct pool_head *pool)
 {
 	int ret = 0;
@@ -1185,7 +1206,7 @@ INITCALL0(STG_PREPARE, init_pools);
 /* Report in build options if trim is supported */
 static void pools_register_build_options(void)
 {
-	if (is_trim_enabled()) {
+	if (is_trim_enabled() && _malloc_trim) {
 		char *ptr = NULL;
 		memprintf(&ptr, "Support for malloc_trim() is enabled.");
 		hap_register_build_opts(ptr, 1);