diff --git a/src/raw_sock.c b/src/raw_sock.c
index 9298ed780..52a48a5e4 100644
--- a/src/raw_sock.c
+++ b/src/raw_sock.c
@@ -156,10 +156,13 @@ int raw_sock_to_pipe(struct connection *conn, struct pipe *pipe, unsigned int co
 		}
 	} /* while */
 
+	if (unlikely(conn->flags & CO_FL_WAIT_L4_CONN) && retval)
+		conn->flags &= ~CO_FL_WAIT_L4_CONN;
 	return retval;
 
  out_read0:
 	conn_sock_read0(conn);
+	conn->flags &= ~CO_FL_WAIT_L4_CONN;
 	return retval;
 }
 
@@ -190,6 +193,8 @@ int raw_sock_from_pipe(struct connection *conn, struct pipe *pipe)
 		done += ret;
 		pipe->data -= ret;
 	}
+	if (unlikely(conn->flags & CO_FL_WAIT_L4_CONN) && done)
+		conn->flags &= ~CO_FL_WAIT_L4_CONN;
 	return done;
 }
 
@@ -269,10 +274,14 @@ static int raw_sock_to_buf(struct connection *conn, struct buffer *buf, int coun
 			break;
 		}
 	}
+
+	if (unlikely(conn->flags & CO_FL_WAIT_L4_CONN) && done)
+		conn->flags &= ~CO_FL_WAIT_L4_CONN;
 	return done;
 
  read0:
 	conn_sock_read0(conn);
+	conn->flags &= ~CO_FL_WAIT_L4_CONN;
 	return done;
 }
 
@@ -330,6 +339,8 @@ static int raw_sock_from_buf(struct connection *conn, struct buffer *buf, int fl
 			break;
 		}
 	}
+	if (unlikely(conn->flags & CO_FL_WAIT_L4_CONN) && done)
+		conn->flags &= ~CO_FL_WAIT_L4_CONN;
 	return done;
 }
 
diff --git a/src/stream_interface.c b/src/stream_interface.c
index 8bc5bbbb2..e857c129b 100644
--- a/src/stream_interface.c
+++ b/src/stream_interface.c
@@ -721,9 +721,6 @@ static int si_conn_send_loop(struct connection *conn)
 		if (ret <= 0)
 			break;
 
-		if (si->conn.flags & CO_FL_WAIT_L4_CONN)
-			si->conn.flags &= ~CO_FL_WAIT_L4_CONN;
-
 		b->flags |= CF_WRITE_PARTIAL;
 
 		if (!b->buf.o) {
@@ -1049,9 +1046,6 @@ static void si_conn_recv_cb(struct connection *conn)
 			b_adv(&b->buf, fwd);
 		}
 
-		if (conn->flags & CO_FL_WAIT_L4_CONN)
-			conn->flags &= ~CO_FL_WAIT_L4_CONN;
-
 		b->flags |= CF_READ_PARTIAL;
 		b->total += ret;