diff --git a/src/haproxy.c b/src/haproxy.c
index c3b7be649..2675bf715 100644
--- a/src/haproxy.c
+++ b/src/haproxy.c
@@ -2835,11 +2835,12 @@ void run_poll_loop()
 			int i;
 
 			if (stopping) {
-				_HA_ATOMIC_OR(&stopping_thread_mask, tid_bit);
-				/* notify all threads that stopping was just set */
-				for (i = 0; i < global.nbthread; i++)
-					if (((all_threads_mask & ~stopping_thread_mask) >> i) & 1)
-						wake_thread(i);
+				if (_HA_ATOMIC_OR(&stopping_thread_mask, tid_bit) == tid_bit) {
+					/* notify all threads that stopping was just set */
+					for (i = 0; i < global.nbthread; i++)
+						if ((all_threads_mask >> i) & 1)
+							wake_thread(i);
+				}
 			}
 
 			/* stop when there's nothing left to do */