diff --git a/src/proto_quic.c b/src/proto_quic.c
index 34e22b8dc..cf1e0ded1 100644
--- a/src/proto_quic.c
+++ b/src/proto_quic.c
@@ -592,7 +592,8 @@ static int quic_alloc_rxbufs_listener(struct listener *l)
  */
 static int quic_bind_listener(struct listener *listener, char *errmsg, int errlen)
 {
-	int err = ERR_NONE;
+	const struct sockaddr_storage addr = listener->rx.addr;
+	int fd, err = ERR_NONE;
 	char *msg = NULL;
 
 	/* ensure we never return garbage */
@@ -607,6 +608,25 @@ static int quic_bind_listener(struct listener *listener, char *errmsg, int errle
 		goto udp_return;
 	}
 
+	/* Set IP_PKTINFO to retrieve destination address on recv. */
+	fd = listener->rx.fd;
+	switch (addr.ss_family) {
+	case AF_INET:
+#if defined(IP_PKTINFO)
+		setsockopt(fd, IPPROTO_IP, IP_PKTINFO, &one, sizeof(one));
+#elif defined(IP_RECVDSTADDR)
+		setsockopt(fd, IPPROTO_IP, IP_RECVDSTADDR, &one, sizeof(one));
+#endif /* IP_PKTINFO || IP_RECVDSTADDR */
+		break;
+	case AF_INET6:
+#ifdef IPV6_RECVPKTINFO
+		setsockopt(fd, IPPROTO_IPV6, IPV6_RECVPKTINFO, &one, sizeof(one));
+#endif
+		break;
+	default:
+		break;
+	}
+
 	if (!quic_alloc_rxbufs_listener(listener)) {
 		msg = "could not initialize tx/rx rings";
 		err |= ERR_WARN;
diff --git a/src/sock_inet.c b/src/sock_inet.c
index f8e872cbf..d517e4c42 100644
--- a/src/sock_inet.c
+++ b/src/sock_inet.c
@@ -381,25 +381,6 @@ int sock_inet_bind_receiver(struct receiver *rx, char **errmsg)
 	}
 #endif
 
-#ifdef USE_QUIC
-	if (rx->proto->proto_type == PROTO_TYPE_DGRAM) {
-		switch (addr_inet.ss_family) {
-		case AF_INET:
-#if defined(IP_PKTINFO)
-			setsockopt(fd, IPPROTO_IP, IP_PKTINFO, &one, sizeof(one));
-#elif defined(IP_RECVDSTADDR)
-			setsockopt(fd, IPPROTO_IP, IP_RECVDSTADDR, &one, sizeof(one));
-#endif /* IP_PKTINFO || IP_RECVDSTADDR */
-			break;
-		case AF_INET6:
-#ifdef IPV6_RECVPKTINFO
-			setsockopt(fd, IPPROTO_IPV6, IPV6_RECVPKTINFO, &one, sizeof(one));
-#endif
-			break;
-		}
-	}
-#endif /* USE_QUIC */
-
 	if (!ext && bind(fd, (struct sockaddr *)&addr_inet, rx->proto->fam->sock_addrlen) == -1) {
 		err |= ERR_RETRYABLE | ERR_ALERT;
 		memprintf(errmsg, "cannot bind socket (%s)", strerror(errno));