diff --git a/include/proto/session.h b/include/proto/session.h
index 2ff8e382d..eabebb85a 100644
--- a/include/proto/session.h
+++ b/include/proto/session.h
@@ -124,12 +124,12 @@ static inline int session_check_idle_conn(struct session *sess, struct connectio
 		/* We can't keep the connection, let's try to add it to the server idle list */
 		session_unown_conn(sess, conn);
 		conn->owner = NULL;
+		conn->flags &= ~CO_FL_SESS_IDLE;
 		if (!srv_add_to_idle_list(objt_server(conn->target), conn)) {
 			/* The server doesn't want it, let's kill the connection right away */
 			conn->mux->destroy(conn->ctx);
 			return -1;
-		} else
-			conn->flags &= ~CO_FL_SESS_IDLE;
+		}
 		return 1;
 	} else {
 		conn->flags |= CO_FL_SESS_IDLE;
diff --git a/src/backend.c b/src/backend.c
index d7695bf2c..e41689d2a 100644
--- a/src/backend.c
+++ b/src/backend.c
@@ -1406,6 +1406,7 @@ int connect_server(struct stream *s)
 				session_unown_conn(s->sess, old_conn);
 				old_conn->owner = sess;
 				if (!session_add_conn(sess, old_conn, old_conn->target)) {
+					old_conn->flags &= ~CO_FL_SESS_IDLE;
 					old_conn->owner = NULL;
 					old_conn->mux->destroy(old_conn->ctx);
 				} else