diff --git a/src/backend.c b/src/backend.c
index a9e20e078..cfd24625c 100644
--- a/src/backend.c
+++ b/src/backend.c
@@ -1335,8 +1335,8 @@ int connect_server(struct stream *s)
 				reuse = 0;
 		}
 	}
-	if ((!reuse || (srv_conn && !(srv_conn->flags & CO_FL_CONNECTED)))
-	    && ha_used_fds > global.tune.pool_high_count) {
+	if (((!reuse || (srv_conn && !(srv_conn->flags & CO_FL_CONNECTED)))
+	    && ha_used_fds > global.tune.pool_high_count) && srv->idle_orphan_conns) {
 		struct connection *tokill_conn;
 
 		/* We can't reuse a connection, and e have more FDs than deemd