diff --git a/src/listener.c b/src/listener.c
index b2a6862ca..39f5b340f 100644
--- a/src/listener.c
+++ b/src/listener.c
@@ -362,7 +362,6 @@ int pause_listener(struct listener *l)
 
 	MT_LIST_DEL(&l->wait_queue);
 
-	fd_stop_recv(l->rx.fd);
 	listener_set_state(l, LI_PAUSED);
 
 	if (px && !px->li_ready) {
diff --git a/src/proto_tcp.c b/src/proto_tcp.c
index 83b23b9b5..b9bc97e42 100644
--- a/src/proto_tcp.c
+++ b/src/proto_tcp.c
@@ -743,6 +743,7 @@ int tcp_pause_listener(struct listener *l)
 	if (shutdown(l->rx.fd, SHUT_RD) != 0)
 		goto check_already_done; /* show always be OK */
 
+	fd_stop_recv(l->rx.fd);
 	return 1;
 
  check_already_done:
@@ -752,11 +753,15 @@ int tcp_pause_listener(struct listener *l)
 	 */
 	opt_val = 0;
 	opt_len = sizeof(opt_val);
-	if (getsockopt(l->rx.fd, SOL_SOCKET, SO_ACCEPTCONN, &opt_val, &opt_len) == -1)
+	if (getsockopt(l->rx.fd, SOL_SOCKET, SO_ACCEPTCONN, &opt_val, &opt_len) == -1) {
+		fd_stop_recv(l->rx.fd);
 		return 0; /* the socket is really unrecoverable */
+	}
 
-	if (!opt_val)
+	if (!opt_val) {
+		fd_stop_recv(l->rx.fd);
 		return 1; /* already paused by another process */
+	}
 
 	/* something looks fishy here */
 	return -1;
diff --git a/src/proto_uxst.c b/src/proto_uxst.c
index bafe97dcf..c477d193d 100644
--- a/src/proto_uxst.c
+++ b/src/proto_uxst.c
@@ -151,7 +151,8 @@ static void uxst_add_listener(struct listener *listener, int port)
 /* Pause a listener. Returns < 0 in case of failure, 0 if the listener
  * was totally stopped, or > 0 if correctly paused. Nothing is done for
  * plain unix sockets since currently it's the new process which handles
- * the renaming. Abstract sockets are completely unbound.
+ * the renaming. Abstract sockets are completely unbound and closed so
+ * there's no need to stop the poller.
  */
 static int uxst_pause_listener(struct listener *l)
 {