diff --git a/storage/remote/client.go b/storage/remote/client.go index 764463e3e4..7b81501b1e 100644 --- a/storage/remote/client.go +++ b/storage/remote/client.go @@ -180,6 +180,17 @@ func newRemoteReadMetrics( return readQueries, readQueriesTotalVec, readQueryDurationVec } +// Unregister metrics for a removed remote_read config. +func unregisterRemoteReadMetrics(key string) { + if val, ok := remoteReadMetricCache.LoadAndDelete(key); ok { + if m, ok := val.(*readClientMetrics); ok && m.reg != nil { + m.reg.Unregister(m.readQueries) + m.reg.Unregister(m.readQueriesTotalVec) + m.reg.Unregister(m.readQueryDurationVec) + } + } +} + // NewReadClient creates a new client for remote read. func NewReadClient(name string, conf *ClientConfig, reg prometheus.Registerer, optFuncs ...config_util.HTTPClientOption) (ReadClient, error) { httpClient, err := config_util.NewClientFromConfig(conf.HTTPClientConfig, "remote_storage_read_client", optFuncs...) diff --git a/storage/remote/storage.go b/storage/remote/storage.go index 0f99f87536..18b766d6e6 100644 --- a/storage/remote/storage.go +++ b/storage/remote/storage.go @@ -61,6 +61,8 @@ type Storage struct { // For reads. queryables []storage.SampleAndChunkQueryable localStartTimeCallback startTimeCallback + + readRemotes map[string]struct{} } // NewStorage returns a remote.Storage. @@ -79,6 +81,7 @@ func NewStorage(l *slog.Logger, reg prometheus.Registerer, stCallback startTimeC logger: logger, deduper: deduper, localStartTimeCallback: stCallback, + readRemotes: make(map[string]struct{}), } s.rws = NewWriteStorage(s.logger, reg, walDir, flushDeadline, sm) return s @@ -97,9 +100,13 @@ func (s *Storage) ApplyConfig(conf *config.Config) error { return err } + // Track active remote_read keys in the current config. + activeReadRemotes := make(map[string]struct{}) + // Update read clients readHashes := make(map[string]struct{}) queryables := make([]storage.SampleAndChunkQueryable, 0, len(conf.RemoteReadConfigs)) + for _, rrConf := range conf.RemoteReadConfigs { hash, err := toHash(rrConf) if err != nil { @@ -112,14 +119,16 @@ func (s *Storage) ApplyConfig(conf *config.Config) error { } readHashes[hash] = struct{}{} - // Set the queue name to the config hash if the user has not set - // a name in their remote write config so we can still differentiate - // between queues that have the same remote write endpoint. + // Generate the remote name. name := hash[:6] if rrConf.Name != "" { name = rrConf.Name } + // Generate the remote key and track it. + key := name + "|" + rrConf.URL.String() + activeReadRemotes[key] = struct{}{} + c, err := NewReadClient(name, &ClientConfig{ URL: rrConf.URL, Timeout: rrConf.RemoteTimeout, @@ -135,6 +144,7 @@ func (s *Storage) ApplyConfig(conf *config.Config) error { if !rrConf.FilterExternalLabels { externalLabels = labels.EmptyLabels() } + queryables = append(queryables, NewSampleAndChunkQueryableClient( c, externalLabels, @@ -143,6 +153,16 @@ func (s *Storage) ApplyConfig(conf *config.Config) error { s.localStartTimeCallback, )) } + + // Unregister metrics for any remote_read configs that were removed + for oldKey := range s.readRemotes { + if _, stillExists := activeReadRemotes[oldKey]; !stillExists { + unregisterRemoteReadMetrics(oldKey) + } + } + + // Save new state + s.readRemotes = activeReadRemotes s.queryables = queryables return nil