diff --git a/include/haproxy/thread.h b/include/haproxy/thread.h
index af909f170..e7d6d2cba 100644
--- a/include/haproxy/thread.h
+++ b/include/haproxy/thread.h
@@ -265,7 +265,7 @@ static inline void thread_harmless_end()
 {
 	while (1) {
 		HA_ATOMIC_AND(&threads_harmless_mask, ~tid_bit);
-		if (likely((threads_want_rdv_mask & ~tid_bit) == 0))
+		if (likely((threads_want_rdv_mask & all_threads_mask & ~tid_bit) == 0))
 			break;
 		thread_harmless_till_end();
 	}
diff --git a/src/thread.c b/src/thread.c
index 742e47c09..07017dd18 100644
--- a/src/thread.c
+++ b/src/thread.c
@@ -56,7 +56,7 @@ struct lock_stat lock_stats[LOCK_LABELS];
 void thread_harmless_till_end()
 {
 		_HA_ATOMIC_OR(&threads_harmless_mask, tid_bit);
-		while (threads_want_rdv_mask & ~tid_bit) {
+		while (threads_want_rdv_mask & all_threads_mask & ~tid_bit) {
 			ha_thread_relax();
 		}
 }