on removal of remote_read entry remove its metrics as well

Signed-off-by: Vandit Singh <vanditsinghkv@gmail.com>

storage/remote/client.go

@@ -180,6 +180,17 @@ func newRemoteReadMetrics(
 	return readQueries, readQueriesTotalVec, readQueryDurationVec
 }
 
+// Unregister metrics for a removed remote_read config.
+func unregisterRemoteReadMetrics(key string) {
+	if val, ok := remoteReadMetricCache.LoadAndDelete(key); ok {
+		if m, ok := val.(*readClientMetrics); ok && m.reg != nil {
+			m.reg.Unregister(m.readQueries)
+			m.reg.Unregister(m.readQueriesTotalVec)
+			m.reg.Unregister(m.readQueryDurationVec)
+		}
+	}
+}
+
 // NewReadClient creates a new client for remote read.
 func NewReadClient(name string, conf *ClientConfig, reg prometheus.Registerer, optFuncs ...config_util.HTTPClientOption) (ReadClient, error) {
 	httpClient, err := config_util.NewClientFromConfig(conf.HTTPClientConfig, "remote_storage_read_client", optFuncs...)

storage/remote/storage.go

@@ -61,6 +61,8 @@ type Storage struct {
 	// For reads.
 	queryables             []storage.SampleAndChunkQueryable
 	localStartTimeCallback startTimeCallback
+
+	readRemotes map[string]struct{}
 }
 
 // NewStorage returns a remote.Storage.
@@ -79,6 +81,7 @@ func NewStorage(l *slog.Logger, reg prometheus.Registerer, stCallback startTimeC
 		logger:                 logger,
 		deduper:                deduper,
 		localStartTimeCallback: stCallback,
+		readRemotes:            make(map[string]struct{}),
 	}
 	s.rws = NewWriteStorage(s.logger, reg, walDir, flushDeadline, sm)
 	return s
@@ -97,9 +100,13 @@ func (s *Storage) ApplyConfig(conf *config.Config) error {
 		return err
 	}
 
+	// Track active remote_read keys in the current config.
+	activeReadRemotes := make(map[string]struct{})
+
 	// Update read clients
 	readHashes := make(map[string]struct{})
 	queryables := make([]storage.SampleAndChunkQueryable, 0, len(conf.RemoteReadConfigs))
+
 	for _, rrConf := range conf.RemoteReadConfigs {
 		hash, err := toHash(rrConf)
 		if err != nil {
@@ -112,14 +119,16 @@ func (s *Storage) ApplyConfig(conf *config.Config) error {
 		}
 		readHashes[hash] = struct{}{}
 
-		// Set the queue name to the config hash if the user has not set
+		// Generate the remote name.
-		// a name in their remote write config so we can still differentiate
-		// between queues that have the same remote write endpoint.
 		name := hash[:6]
 		if rrConf.Name != "" {
 			name = rrConf.Name
 		}
 
+		// Generate the remote key and track it.
+		key := name + "|" + rrConf.URL.String()
+		activeReadRemotes[key] = struct{}{}
+
 		c, err := NewReadClient(name, &ClientConfig{
 			URL:              rrConf.URL,
 			Timeout:          rrConf.RemoteTimeout,
@@ -135,6 +144,7 @@ func (s *Storage) ApplyConfig(conf *config.Config) error {
 		if !rrConf.FilterExternalLabels {
 			externalLabels = labels.EmptyLabels()
 		}
+
 		queryables = append(queryables, NewSampleAndChunkQueryableClient(
 			c,
 			externalLabels,
@@ -143,6 +153,16 @@ func (s *Storage) ApplyConfig(conf *config.Config) error {
 			s.localStartTimeCallback,
 		))
 	}
+
+	// Unregister metrics for any remote_read configs that were removed
+	for oldKey := range s.readRemotes {
+		if _, stillExists := activeReadRemotes[oldKey]; !stillExists {
+			unregisterRemoteReadMetrics(oldKey)
+		}
+	}
+
+	// Save new state
+	s.readRemotes = activeReadRemotes
 	s.queryables = queryables
 
 	return nil