diff --git a/pkg/udp/conn_test.go b/pkg/udp/conn_test.go
index 0876653ce..3e9459102 100644
--- a/pkg/udp/conn_test.go
+++ b/pkg/udp/conn_test.go
@@ -250,13 +250,19 @@ func TestShutdown(t *testing.T) {
 	// Start sending packets, to create a "session" with the server.
 	requireEcho(t, "TEST", conn, time.Second)
 
+	shutdownStartedChan := make(chan struct{})
 	doneChan := make(chan struct{})
 	go func() {
+		close(shutdownStartedChan)
 		err := l.Shutdown(5 * time.Second)
 		require.NoError(t, err)
 		close(doneChan)
 	}()
 
+	// Wait until shutdown has started, and hopefully after 100 ms the listener has stopped accepting new sessions.
+	<-shutdownStartedChan
+	time.Sleep(100 * time.Millisecond)
+
 	// Make sure that our session is still live even after the shutdown.
 	requireEcho(t, "TEST2", conn, time.Second)