diff --git a/src/quic_sock.c b/src/quic_sock.c
index d7149ab84..0be96b605 100644
--- a/src/quic_sock.c
+++ b/src/quic_sock.c
@@ -688,6 +688,23 @@ static void cmsg_set_gso(struct msghdr *msg, struct cmsghdr **cmsg,
 #endif
 }
 
+/* Return 0 if the source address may be used, something not null
+ * if not
+ */
+static int qc_may_use_saddr(struct quic_conn *qc)
+{
+	/* For QUIC backends, the connection fd is always initialized */
+	/* Not useful for connection using the listener socket */
+	if (qc_test_fd(qc))
+		return 0;
+
+	/* Connection to a listener from here.
+	 * Set source address when using listener socket if possible.
+	 */
+	return (!is_addr(&__objt_listener(qc->target)->rx.addr) &&
+	        is_addr(&qc->local_addr));
+}
+
 /* Send a datagram stored into <buf> buffer with <sz> as size. The caller must
  * ensure there is at least <sz> bytes in this buffer.
  *
@@ -760,8 +777,7 @@ int qc_snd_buf(struct quic_conn *qc, const struct buffer *buf, size_t sz,
 	if (qc_test_fd(qc) && !fd_send_ready(qc->fd))
 		return 0;
 
-	/* Set source address when using listener socket if possible. */
-	if (!qc_test_fd(qc) && is_addr(&qc->local_addr)) {
+	if (qc_may_use_saddr(qc)) {
 		msg.msg_control = ancillary_data.bufaddr;
 		cmsg_set_saddr(&msg, &cmsg, &qc->local_addr);
 	}