diff --git a/src/check.c b/src/check.c
index 0f7d19859..f39f88dd9 100644
--- a/src/check.c
+++ b/src/check.c
@@ -1786,8 +1786,11 @@ int init_srv_agent_check(struct server *srv)
 
 static void deinit_srv_check(struct server *srv)
 {
-	if (srv->check.state & CHK_ST_CONFIGURED)
+	if (srv->check.state & CHK_ST_CONFIGURED) {
 		free_check(&srv->check);
+		/* it is safe to drop now since the main server reference is still held by the proxy */
+		srv_drop(srv);
+	}
 	srv->check.state &= ~CHK_ST_CONFIGURED & ~CHK_ST_ENABLED;
 	srv->do_check = 0;
 }
@@ -1795,8 +1798,11 @@ static void deinit_srv_check(struct server *srv)
 
 static void deinit_srv_agent_check(struct server *srv)
 {
-	if (srv->agent.state & CHK_ST_CONFIGURED)
+	if (srv->agent.state & CHK_ST_CONFIGURED) {
 		free_check(&srv->agent);
+		/* it is safe to drop now since the main server reference is still held by the proxy */
+		srv_drop(srv);
+	}
 
 	srv->agent.state &= ~CHK_ST_CONFIGURED & ~CHK_ST_ENABLED & ~CHK_ST_AGENT;
 	srv->do_agent = 0;
diff --git a/src/server.c b/src/server.c
index 9c1228812..f18fc5b2b 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2408,7 +2408,7 @@ void srv_take(struct server *srv)
 
 /* Deallocate a server <srv> and its member. <srv> must be allocated. For
  * dynamic servers, its refcount is decremented first. The free operations are
- * conducted only if the refcount is nul, unless the process is stopping.
+ * conducted only if the refcount is nul.
  *
  * As a convenience, <srv.next> is returned if srv is not NULL. It may be useful
  * when calling srv_drop on the list of servers.
@@ -2425,10 +2425,8 @@ struct server *srv_drop(struct server *srv)
 	/* For dynamic servers, decrement the reference counter. Only free the
 	 * server when reaching zero.
 	 */
-	if (likely(!(global.mode & MODE_STOPPING))) {
-		if (HA_ATOMIC_SUB_FETCH(&srv->refcount, 1))
-			goto end;
-	}
+	if (HA_ATOMIC_SUB_FETCH(&srv->refcount, 1))
+		goto end;
 
 	/* make sure we are removed from our 'next->prev_deleted' list
 	 * This doesn't require full thread isolation as we're using mt lists