diff --git a/src/connection.c b/src/connection.c
index 696525d3e..089ad8925 100644
--- a/src/connection.c
+++ b/src/connection.c
@@ -42,5 +42,7 @@ int conn_fd_handler(int fd)
 		if (!conn->data->write(fd))
 			ret |= FD_WAIT_WRITE;
 
+	/* remove the events before leaving */
+	fdtab[fd].ev &= ~(FD_POLL_IN | FD_POLL_OUT | FD_POLL_HUP | FD_POLL_ERR);
 	return ret;
 }
diff --git a/src/sock_raw.c b/src/sock_raw.c
index 90081aa33..ac0dbcde2 100644
--- a/src/sock_raw.c
+++ b/src/sock_raw.c
@@ -484,7 +484,6 @@ static int sock_raw_read(int fd)
 	if (b->flags & BF_READ_ACTIVITY)
 		b->flags &= ~BF_READ_DONTWAIT;
 
-	fdtab[fd].ev &= ~FD_POLL_IN;
 	return retval;
 
  out_shutdown_r:
@@ -741,7 +740,6 @@ static int sock_raw_write(int fd)
 			task_wakeup(si->owner, TASK_WOKEN_IO);
 	}
 
-	fdtab[fd].ev &= ~FD_POLL_OUT;
 	return retval;
 
  out_error: