diff --git a/dev/flags/flags.c b/dev/flags/flags.c
index 2164a352b..3bcbb7f28 100644
--- a/dev/flags/flags.c
+++ b/dev/flags/flags.c
@@ -163,8 +163,6 @@ void show_conn_flags(unsigned int f)
 	SHOW_FLAG(f, CO_FL_SOCKS4_SEND);
 	SHOW_FLAG(f, CO_FL_EARLY_DATA);
 	SHOW_FLAG(f, CO_FL_EARLY_SSL_HS);
-	SHOW_FLAG(f, CO_FL_ADDR_TO_SET);
-	SHOW_FLAG(f, CO_FL_ADDR_FROM_SET);
 	SHOW_FLAG(f, CO_FL_WAIT_ROOM);
 	SHOW_FLAG(f, CO_FL_WANT_DRAIN);
 	SHOW_FLAG(f, CO_FL_XPRT_READY);
diff --git a/include/haproxy/connection-t.h b/include/haproxy/connection-t.h
index 22160f3f3..77e31eb46 100644
--- a/include/haproxy/connection-t.h
+++ b/include/haproxy/connection-t.h
@@ -100,8 +100,8 @@ enum {
 	CO_FL_WAIT_ROOM     = 0x00000800,  /* data sink is full */
 
 	/* These flags are used to report whether the from/to addresses are set or not */
-	CO_FL_ADDR_FROM_SET = 0x00001000,  /* addr.from is set */
-	CO_FL_ADDR_TO_SET   = 0x00002000,  /* addr.to is set */
+	/* unused: 0x00001000 */
+	/* unused: 0x00002000 */
 
 	CO_FL_EARLY_SSL_HS  = 0x00004000,  /* We have early data pending, don't start SSL handshake yet */
 	CO_FL_EARLY_DATA    = 0x00008000,  /* At least some of the data are early data */
diff --git a/include/haproxy/connection.h b/include/haproxy/connection.h
index 8240a6d40..30e689e15 100644
--- a/include/haproxy/connection.h
+++ b/include/haproxy/connection.h
@@ -327,17 +327,13 @@ static inline void conn_force_unsubscribe(struct connection *conn)
 /* Returns the source address of the connection or NULL if not set */
 static inline const struct sockaddr_storage *conn_src(struct connection *conn)
 {
-	if (conn->flags & CO_FL_ADDR_FROM_SET)
-		return conn->src;
-	return NULL;
+	return conn->src;
 }
 
 /* Returns the destination address of the connection or NULL if not set */
 static inline const struct sockaddr_storage *conn_dst(struct connection *conn)
 {
-	if (conn->flags & CO_FL_ADDR_TO_SET)
-		return conn->dst;
-	return NULL;
+	return conn->dst;
 }
 
 /* Retrieves the connection's original source address. Returns non-zero on
@@ -346,7 +342,7 @@ static inline const struct sockaddr_storage *conn_dst(struct connection *conn)
  */
 static inline int conn_get_src(struct connection *conn)
 {
-	if (conn->flags & CO_FL_ADDR_FROM_SET)
+	if (conn->src)
 		return 1;
 
 	if (!conn_ctrl_ready(conn))
@@ -375,7 +371,6 @@ static inline int conn_get_src(struct connection *conn)
 	sockaddr_free(&conn->src);
 	return 0;
  done:
-	conn->flags |= CO_FL_ADDR_FROM_SET;
 	return 1;
 }
 
@@ -385,7 +380,7 @@ static inline int conn_get_src(struct connection *conn)
  */
 static inline int conn_get_dst(struct connection *conn)
 {
-	if (conn->flags & CO_FL_ADDR_TO_SET)
+	if (conn->dst)
 		return 1;
 
 	if (!conn_ctrl_ready(conn))
@@ -414,7 +409,6 @@ static inline int conn_get_dst(struct connection *conn)
 	sockaddr_free(&conn->dst);
 	return 0;
  done:
-	conn->flags |= CO_FL_ADDR_TO_SET;
 	return 1;
 }
 
diff --git a/src/connection.c b/src/connection.c
index abe4fd66e..e3e1bc876 100644
--- a/src/connection.c
+++ b/src/connection.c
@@ -2088,10 +2088,10 @@ int conn_append_debug_info(struct buffer *buf, const struct connection *conn, co
 
 	chunk_appendf(buf, " %s/%s", conn_get_xprt_name(conn), conn_get_ctrl_name(conn));
 
-	if (conn->flags & CO_FL_ADDR_FROM_SET && addr_to_str(conn->src, addr, sizeof(addr)))
+	if (conn->src && addr_to_str(conn->src, addr, sizeof(addr)))
 		chunk_appendf(buf, " src=%s:%d", addr, get_host_port(conn->src));
 
-	if (conn->flags & CO_FL_ADDR_TO_SET && addr_to_str(conn->dst, addr, sizeof(addr)))
+	if (conn->dst && addr_to_str(conn->dst, addr, sizeof(addr)))
 		chunk_appendf(buf, " dst=%s:%d", addr, get_host_port(conn->dst));
 
 	return buf->data - old_len;
diff --git a/src/proto_quic.c b/src/proto_quic.c
index 5d9240cb7..5c2004527 100644
--- a/src/proto_quic.c
+++ b/src/proto_quic.c
@@ -505,8 +505,6 @@ int quic_connect_server(struct connection *conn, int flags)
 		conn->flags &= ~CO_FL_WAIT_L4_CONN;
 	}
 
-	conn->flags |= CO_FL_ADDR_TO_SET;
-
 	conn_ctrl_init(conn);       /* registers the FD */
 	HA_ATOMIC_OR(&fdtab[fd].state, FD_LINGER_RISK);  /* close hard if needed */
 
diff --git a/src/proto_sockpair.c b/src/proto_sockpair.c
index 55d31febc..9126fe63d 100644
--- a/src/proto_sockpair.c
+++ b/src/proto_sockpair.c
@@ -352,8 +352,6 @@ static int sockpair_connect_server(struct connection *conn, int flags)
 
 	conn->flags &= ~CO_FL_WAIT_L4_CONN;
 
-	conn->flags |= CO_FL_ADDR_TO_SET;
-
 	/* Prepare to send a few handshakes related to the on-wire protocol. */
 	if (conn->send_proxy_ofs)
 		conn->flags |= CO_FL_SEND_PROXY;
@@ -484,7 +482,6 @@ struct connection *sockpair_accept_conn(struct listener *l, int *status)
 		/* just like with UNIX sockets, only the family is filled */
 		conn->src->ss_family = AF_UNIX;
 		conn->handle.fd = cfd;
-		conn->flags |= CO_FL_ADDR_FROM_SET;
 		ret = CO_AC_DONE;
 		goto done;
 	}
diff --git a/src/proto_tcp.c b/src/proto_tcp.c
index 77433b0e5..42342d652 100644
--- a/src/proto_tcp.c
+++ b/src/proto_tcp.c
@@ -552,8 +552,6 @@ int tcp_connect_server(struct connection *conn, int flags)
 		conn->flags &= ~CO_FL_WAIT_L4_CONN;
 	}
 
-	conn->flags |= CO_FL_ADDR_TO_SET;
-
 	conn_ctrl_init(conn);       /* registers the FD */
 	HA_ATOMIC_OR(&fdtab[fd].state, FD_LINGER_RISK);  /* close hard if needed */
 
diff --git a/src/proto_uxst.c b/src/proto_uxst.c
index da39e6980..2a32fa677 100644
--- a/src/proto_uxst.c
+++ b/src/proto_uxst.c
@@ -336,8 +336,6 @@ static int uxst_connect_server(struct connection *conn, int flags)
 		conn->flags &= ~CO_FL_WAIT_L4_CONN;
 	}
 
-	conn->flags |= CO_FL_ADDR_TO_SET;
-
 	/* Prepare to send a few handshakes related to the on-wire protocol. */
 	if (conn->send_proxy_ofs)
 		conn->flags |= CO_FL_SEND_PROXY;
diff --git a/src/quic_sock.c b/src/quic_sock.c
index b6d2d18dc..a0f8129f4 100644
--- a/src/quic_sock.c
+++ b/src/quic_sock.c
@@ -157,7 +157,7 @@ static int new_quic_cli_conn(struct quic_conn *qc, struct listener *l,
 	if (!sockaddr_alloc(&cli_conn->src, saddr, sizeof *saddr))
 		goto out_free_conn;
 
-	cli_conn->flags |= CO_FL_ADDR_FROM_SET | CO_FL_FDLESS;
+	cli_conn->flags |= CO_FL_FDLESS;
 	qc->conn = cli_conn;
 	cli_conn->handle.qc = qc;
 
diff --git a/src/sock.c b/src/sock.c
index 6f58aec2f..602e9c55b 100644
--- a/src/sock.c
+++ b/src/sock.c
@@ -116,7 +116,6 @@ struct connection *sock_accept_conn(struct listener *l, int *status)
 
 		conn->src = addr;
 		conn->handle.fd = cfd;
-		conn->flags |= CO_FL_ADDR_FROM_SET;
 		ret = CO_AC_DONE;
 		goto done;
 	}