diff --git a/include/proto/connection.h b/include/proto/connection.h
index a36906d18..be9d712f6 100644
--- a/include/proto/connection.h
+++ b/include/proto/connection.h
@@ -434,6 +434,11 @@ static inline void conn_sock_read0(struct connection *c)
 {
 	c->flags |= CO_FL_SOCK_RD_SH;
 	__conn_sock_stop_recv(c);
+	/* we don't risk keeping ports unusable if we found the
+	 * zero from the other side.
+	 */
+	if (c->flags & CO_FL_CTRL_READY)
+		fdtab[c->t.sock.fd].linger_risk = 0;
 }
 
 static inline void conn_data_read0(struct connection *c)
diff --git a/include/proto/fd.h b/include/proto/fd.h
index 3b1365d86..7fe616e38 100644
--- a/include/proto/fd.h
+++ b/include/proto/fd.h
@@ -226,6 +226,7 @@ static inline void fd_insert(int fd)
 {
 	fdtab[fd].ev = 0;
 	fdtab[fd].new = 1;
+	fdtab[fd].linger_risk = 0;
 	if (fd + 1 > maxfd)
 		maxfd = fd + 1;
 }
diff --git a/include/types/fd.h b/include/types/fd.h
index 56a8c5c28..701edfc59 100644
--- a/include/types/fd.h
+++ b/include/types/fd.h
@@ -74,6 +74,7 @@ struct fdtab {
 	unsigned char ev;                    /* event seen in return of poll() : FD_POLL_* */
 	unsigned char new:1;                 /* 1 if this fd has just been created */
 	unsigned char updated:1;             /* 1 if this fd is already in the update list */
+	unsigned char linger_risk:1;         /* 1 if we must kill lingering before closing */
 };
 
 /* less often used information */
diff --git a/src/checks.c b/src/checks.c
index 7f195c9ea..8014a668b 100644
--- a/src/checks.c
+++ b/src/checks.c
@@ -1443,10 +1443,9 @@ static int wake_srv_chk(struct connection *conn)
 		/* We're here because nobody wants to handle the error, so we
 		 * sure want to abort the hard way.
 		 */
-		if ((conn->flags & CO_FL_CTRL_READY) && !(conn->flags & CO_FL_SOCK_RD_SH)) {
-			if (conn->flags & CO_FL_WAIT_RD || !conn->ctrl->drain || !conn->ctrl->drain(conn->t.sock.fd))
-				setsockopt(conn->t.sock.fd, SOL_SOCKET, SO_LINGER,
-				           (struct linger *) &nolinger, sizeof(struct linger));
+		if (conn_ctrl_ready(conn) && !(conn->flags & CO_FL_SOCK_RD_SH)) {
+			if (!(conn->flags & CO_FL_WAIT_RD) && conn->ctrl->drain && conn->ctrl->drain(conn->t.sock.fd))
+				fdtab[conn->t.sock.fd].linger_risk = 0;
 		}
 		conn_force_close(conn);
 	}
@@ -1663,10 +1662,9 @@ static struct task *process_chk(struct task *t)
 			 * as a failed response coupled with "observe layer7" caused the
 			 * server state to be suddenly changed.
 			 */
-			if ((conn->flags & CO_FL_CTRL_READY) && !(conn->flags & CO_FL_SOCK_RD_SH)) {
-				if (conn->flags & CO_FL_WAIT_RD || !conn->ctrl->drain || !conn->ctrl->drain(conn->t.sock.fd))
-					setsockopt(conn->t.sock.fd, SOL_SOCKET, SO_LINGER,
-					           (struct linger *) &nolinger, sizeof(struct linger));
+			if (conn_ctrl_ready(conn) && !(conn->flags & CO_FL_SOCK_RD_SH)) {
+				if (!(conn->flags & CO_FL_WAIT_RD) && conn->ctrl->drain && conn->ctrl->drain(conn->t.sock.fd))
+					fdtab[conn->t.sock.fd].linger_risk = 0;
 			}
 			conn_force_close(conn);
 		}
diff --git a/src/fd.c b/src/fd.c
index 5df0b45df..7e691146b 100644
--- a/src/fd.c
+++ b/src/fd.c
@@ -116,6 +116,11 @@ unsigned int *fd_updt = NULL;  // FD updates list
  */
 void fd_delete(int fd)
 {
+	if (fdtab[fd].linger_risk) {
+		/* this is generally set when connecting to servers */
+		setsockopt(fd, SOL_SOCKET, SO_LINGER,
+			   (struct linger *) &nolinger, sizeof(struct linger));
+	}
 	if (cur_poller.clo)
 		cur_poller.clo(fd);
 
diff --git a/src/frontend.c b/src/frontend.c
index 591c1f21a..f94c6ab2b 100644
--- a/src/frontend.c
+++ b/src/frontend.c
@@ -85,8 +85,8 @@ int frontend_accept(struct session *s)
 				   (char *) &one, sizeof(one));
 
 		if (s->fe->options & PR_O_TCP_NOLING)
-			setsockopt(cfd, SOL_SOCKET, SO_LINGER,
-				   (struct linger *) &nolinger, sizeof(struct linger));
+			fdtab[cfd].linger_risk = 1;
+
 #if defined(TCP_MAXSEG)
 		if (s->listener->maxseg < 0) {
 			/* we just want to reduce the current MSS by that value */
diff --git a/src/proto_tcp.c b/src/proto_tcp.c
index 2b6fbc745..c1b0d7bdd 100644
--- a/src/proto_tcp.c
+++ b/src/proto_tcp.c
@@ -475,6 +475,7 @@ int tcp_connect_server(struct connection *conn, int data, int delack)
 		conn->flags |= CO_FL_SEND_PROXY;
 
 	conn_ctrl_init(conn);       /* registers the FD */
+	fdtab[fd].linger_risk = 1;  /* close hard if needed */
 	conn_sock_want_send(conn);  /* for connect status */
 
 	if (conn_xprt_init(conn) < 0) {
diff --git a/src/stream_interface.c b/src/stream_interface.c
index f2650d364..d819ede82 100644
--- a/src/stream_interface.c
+++ b/src/stream_interface.c
@@ -812,10 +812,6 @@ static void stream_int_shutw_conn(struct stream_interface *si)
 			/* quick close, the socket is alredy shut anyway */
 		}
 		else if (si->flags & SI_FL_NOLINGER) {
-			if (conn_ctrl_ready(conn)) {
-				setsockopt(conn->t.sock.fd, SOL_SOCKET, SO_LINGER,
-					   (struct linger *) &nolinger, sizeof(struct linger));
-			}
 			/* unclean data-layer shutdown */
 			if (conn->xprt && conn->xprt->shutw)
 				conn->xprt->shutw(conn, 0);
@@ -1285,12 +1281,6 @@ void stream_sock_read0(struct stream_interface *si)
 
 	if (si->flags & SI_FL_NOHALF) {
 		/* we want to immediately forward this close to the write side */
-		if (si->flags & SI_FL_NOLINGER) {
-			si->flags &= ~SI_FL_NOLINGER;
-			if (conn_ctrl_ready(conn))
-				setsockopt(conn->t.sock.fd, SOL_SOCKET, SO_LINGER,
-				           (struct linger *) &nolinger, sizeof(struct linger));
-		}
 		/* force flag on ssl to keep session in cache */
 		if (conn->xprt->shutw)
 			conn->xprt->shutw(conn, 0);
@@ -1298,6 +1288,8 @@ void stream_sock_read0(struct stream_interface *si)
 	}
 
 	/* otherwise that's just a normal read shutdown */
+	if (conn_ctrl_ready(conn))
+		fdtab[conn->t.sock.fd].linger_risk = 0;
 	__conn_data_stop_recv(conn);
 	return;