diff --git a/src/listener.c b/src/listener.c
index 1ce35de8c..05bb7435a 100644
--- a/src/listener.c
+++ b/src/listener.c
@@ -257,6 +257,9 @@ void listener_accept(int fd)
 	int max_accept = l->maxaccept ? l->maxaccept : 1;
 	int cfd;
 	int ret;
+#ifdef USE_ACCEPT4
+	static int accept4_broken;
+#endif
 
 	if (unlikely(l->nbconn >= l->maxconn)) {
 		listener_full(l);
@@ -346,15 +349,17 @@ void listener_accept(int fd)
 		}
 
 #ifdef USE_ACCEPT4
-		cfd = accept4(fd, (struct sockaddr *)&addr, &laddr, SOCK_NONBLOCK);
-		if (unlikely(cfd == -1 && errno == EINVAL)) {
-			/* unsupported syscall, fallback to normal accept()+fcntl() */
+		/* only call accept4() if it's known to be safe, otherwise
+		 * fallback to the legacy accept() + fcntl().
+		 */
+		if (unlikely(accept4_broken ||
+			((cfd = accept4(fd, (struct sockaddr *)&addr, &laddr, SOCK_NONBLOCK)) == -1 &&
+			(errno == ENOSYS || errno == EINVAL || errno == EBADF) &&
+			(accept4_broken = 1))))
+#endif
 			if ((cfd = accept(fd, (struct sockaddr *)&addr, &laddr)) != -1)
 				fcntl(cfd, F_SETFL, O_NONBLOCK);
-		}
-#else
-		cfd = accept(fd, (struct sockaddr *)&addr, &laddr);
-#endif
+
 		if (unlikely(cfd == -1)) {
 			switch (errno) {
 			case EAGAIN:
diff --git a/src/session.c b/src/session.c
index 1faf0c347..aae0c69f8 100644
--- a/src/session.c
+++ b/src/session.c
@@ -148,14 +148,6 @@ int session_accept(struct listener *l, int cfd, struct sockaddr_storage *addr)
 		goto out_free_session;
 	}
 
-#ifndef USE_ACCEPT4
-	/* Adjust some socket options if the connection was accepted by a plain
-	 * accept() syscall.
-	 */
-	if (unlikely(fcntl(cfd, F_SETFL, O_NONBLOCK) == -1))
-		goto out_free_session;
-#endif
-
 	/* monitor-net and health mode are processed immediately after TCP
 	 * connection rules. This way it's possible to block them, but they
 	 * never use the lower data layers, they send directly over the socket,