diff --git a/include/proto/connection.h b/include/proto/connection.h
index 115e7aaa3..88c7e50e5 100644
--- a/include/proto/connection.h
+++ b/include/proto/connection.h
@@ -1091,6 +1091,9 @@ static inline int conn_install_mux_be(struct connection *conn, void *ctx, struct
 	return conn_install_mux(conn, mux_ops, ctx, prx, sess);
 }
 
+/* Change the mux for the connection.
+ * The caller should make sure he's not subscribed to the underlying XPRT.
+ */
 static inline int conn_upgrade_mux_fe(struct connection *conn, void *ctx, struct buffer *buf,
 				      struct ist mux_proto, int mode)
 {
@@ -1118,7 +1121,6 @@ static inline int conn_upgrade_mux_fe(struct connection *conn, void *ctx, struct
 	old_mux_ctx = conn->ctx;
 	conn->mux = new_mux;
 	conn->ctx = ctx;
-	conn_force_unsubscribe(conn);
 	if (new_mux->init(conn, bind_conf->frontend, conn->owner, buf) == -1) {
 		/* The mux upgrade failed, so restore the old mux */
 		conn->ctx = old_mux_ctx;
diff --git a/src/mux_h1.c b/src/mux_h1.c
index 37cc8252d..e0880e5e8 100644
--- a/src/mux_h1.c
+++ b/src/mux_h1.c
@@ -465,6 +465,10 @@ static void h1_release(struct h1c *h1c)
 
 		if (conn && h1c->flags & H1C_F_UPG_H2C) {
 			h1c->flags &= ~H1C_F_UPG_H2C;
+			/* Make sure we're no longer subscribed to anything */
+			if (h1c->wait_event.events)
+				conn->xprt->unsubscribe(conn, conn->xprt_ctx,
+				    h1c->wait_event.events, &h1c->wait_event);
 			if (conn_upgrade_mux_fe(conn, NULL, &h1c->ibuf, ist("h2"), PROTO_MODE_HTX) != -1) {
 				/* connection successfully upgraded to H2, this
 				 * mux was already released */
diff --git a/src/proxy.c b/src/proxy.c
index 731f5e2a6..329626821 100644
--- a/src/proxy.c
+++ b/src/proxy.c
@@ -1470,6 +1470,13 @@ int stream_set_backend(struct stream *s, struct proxy *be)
 
 			if (conn && cs) {
 				si_rx_endp_more(&s->si[0]);
+				/* Make sure we're unsubscribed, the the new
+				 * mux will probably want to subscribe to
+				 * the underlying XPRT
+				 */
+				if (s->si[0].wait_event.events)
+					conn->mux->unsubscribe(cs, s->si[0].wait_event.events,
+					    &s->si[0].wait_event);
 				if (conn_upgrade_mux_fe(conn, cs, &s->req.buf, ist(""), PROTO_MODE_HTX)  == -1)
 					return 0;
 				s->flags |= SF_HTX;