From b1093c6ba21ae3d41946f3a7bb4886b8b90e7c27 Mon Sep 17 00:00:00 2001
From: Willy Tarreau <w@1wt.eu>
Date: Sat, 9 Jul 2022 18:55:37 +0200
Subject: [PATCH] MEDIUM: poller: program the update in fd_update_events() for
 a migrated FD

When an FD is migrated, all pollers program an update. That's useless
code duplication, and when thread groups will be supported, this will
require an extra round of locking just to verify the update_mask on
return. Let's just program the update direction from fd_update_events()
as it already does for closed FDs, this becomes more logical.
---
 src/ev_epoll.c   |  9 +--------
 src/ev_evports.c |  7 ++-----
 src/ev_kqueue.c  |  9 +--------
 src/ev_poll.c    | 10 ++--------
 src/fd.c         |  4 ++++
 5 files changed, 10 insertions(+), 29 deletions(-)

diff --git a/src/ev_epoll.c b/src/ev_epoll.c
index 1917ed1ac..354a18793 100644
--- a/src/ev_epoll.c
+++ b/src/ev_epoll.c
@@ -214,7 +214,6 @@ static void _do_poll(struct poller *p, int exp, int wake)
 
 	for (count = 0; count < status; count++) {
 		unsigned int n, e;
-		int ret;
 
 		e = epoll_events[count].events;
 		fd = epoll_events[count].data.fd;
@@ -231,13 +230,7 @@ static void _do_poll(struct poller *p, int exp, int wake)
 		    ((e & EPOLLHUP)   ? FD_EV_SHUT_RW : 0) |
 		    ((e & EPOLLERR)   ? FD_EV_ERR_RW  : 0);
 
-		ret = fd_update_events(fd, n);
-
-		if (ret == FD_UPDT_MIGRATED) {
-			/* FD has been migrated */
-			if (!HA_ATOMIC_BTS(&fdtab[fd].update_mask, tid))
-				fd_updt[fd_nbupdt++] = fd;
-		}
+		fd_update_events(fd, n);
 	}
 	/* the caller will take care of cached events */
 }
diff --git a/src/ev_evports.c b/src/ev_evports.c
index 301e86eef..05c9ebcca 100644
--- a/src/ev_evports.c
+++ b/src/ev_evports.c
@@ -244,12 +244,9 @@ static void _do_poll(struct poller *p, int exp, int wake)
 		 */
 		ret = fd_update_events(fd, n);
 
-		/* disable polling on this instance if the FD was migrated */
-		if (ret == FD_UPDT_MIGRATED) {
-			if (!HA_ATOMIC_BTS(&fdtab[fd].update_mask, tid))
-				fd_updt[fd_nbupdt++] = fd;
+		/* polling will be on this instance if the FD was migrated */
+		if (ret == FD_UPDT_MIGRATED)
 			continue;
-		}
 
 		/*
 		 * This file descriptor was closed during the processing of
diff --git a/src/ev_kqueue.c b/src/ev_kqueue.c
index 43643fb38..ff377625b 100644
--- a/src/ev_kqueue.c
+++ b/src/ev_kqueue.c
@@ -178,7 +178,6 @@ static void _do_poll(struct poller *p, int exp, int wake)
 
 	for (count = 0; count < status; count++) {
 		unsigned int n = 0;
-		int ret;
 
 		fd = kev[count].ident;
 
@@ -197,13 +196,7 @@ static void _do_poll(struct poller *p, int exp, int wake)
 				n |= FD_EV_ERR_RW;
 		}
 
-		ret = fd_update_events(fd, n);
-
-		if (ret == FD_UPDT_MIGRATED) {
-			/* FD was migrated, let's stop polling it */
-			if (!HA_ATOMIC_BTS(&fdtab[fd].update_mask, tid))
-				fd_updt[fd_nbupdt++] = fd;
-		}
+		fd_update_events(fd, n);
 	}
 }
 
diff --git a/src/ev_poll.c b/src/ev_poll.c
index 92e45a634..3cc41dc68 100644
--- a/src/ev_poll.c
+++ b/src/ev_poll.c
@@ -213,8 +213,8 @@ static void _do_poll(struct poller *p, int exp, int wake)
 
 	for (count = 0; status > 0 && count < nbfd; count++) {
 		unsigned int n;
-		int ret;
 		int e = poll_events[count].revents;
+
 		fd = poll_events[count].fd;
 
 		if ((e & POLLRDHUP) && !(cur_poller.flags & HAP_POLL_F_RDHUP))
@@ -235,13 +235,7 @@ static void _do_poll(struct poller *p, int exp, int wake)
 		    ((e & POLLHUP)   ? FD_EV_SHUT_RW : 0) |
 		    ((e & POLLERR)   ? FD_EV_ERR_RW  : 0);
 
-		ret = fd_update_events(fd, n);
-
-		if (ret == FD_UPDT_MIGRATED) {
-			/* FD was migrated, let's stop polling it */
-			if (!HA_ATOMIC_BTS(&fdtab[fd].update_mask, tid))
-				fd_updt[fd_nbupdt++] = fd;
-		}
+		fd_update_events(fd, n);
 	}
 }
 
diff --git a/src/fd.c b/src/fd.c
index 29a14135f..ee6cbd2a7 100644
--- a/src/fd.c
+++ b/src/fd.c
@@ -483,6 +483,10 @@ int fd_update_events(int fd, uint evts)
 		if (!(tmask & tid_bit)) {
 			/* a takeover has started */
 			activity[tid].poll_skip_fd++;
+
+			/* Let the poller know this FD was lost */
+			if (!HA_ATOMIC_BTS(&fdtab[fd].update_mask, tid))
+				fd_updt[fd_nbupdt++] = fd;
 			return FD_UPDT_MIGRATED;
 		}
 	} while (!HA_ATOMIC_CAS(&fdtab[fd].running_mask, &rmask, rmask | tid_bit));