mirrors/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2026-05-05 04:16:15 +02:00
Code Issues Projects Releases Wiki Activity

Add counters for unknown series references during WAL/WBL replay

Browse Source 
Signed-off-by: Patryk Prus <p@trykpr.us>
This commit is contained in:
Patryk Prus 2025-03-12 13:11:49 -04:00
parent 85fa39032e
commit 401dbacf2e
No known key found for this signature in database
GPG Key ID: 795650115CA6A58F
2 changed files with 32 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
tsdb/head.go
View File

@ -379,6 +379,8 @@ type headMetrics struct {
snapshotReplayErrorTotal prometheus.Counter // Will be either 0 or 1.
oooHistogram prometheus.Histogram
mmapChunksTotal prometheus.Counter
walReplayUnknownRefsTotal *prometheus.CounterVec
wblReplayUnknownRefsTotal *prometheus.CounterVec
}
const (
@ -510,6 +512,14 @@ func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics {
Name: "prometheus_tsdb_mmap_chunks_total",
Help: "Total number of chunks that were memory-mapped.",
}),
walReplayUnknownRefsTotal: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "prometheus_tsdb_wal_replay_unknown_refs_total",
Help: "Total number of unknown series references encountered during WAL replay.",
}, []string{"type"}),
wblReplayUnknownRefsTotal: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "prometheus_tsdb_wbl_replay_unknown_refs_total",
Help: "Total number of unknown series references encountered during WBL replay.",
}, []string{"type"}),
}
if r != nil {
@ -577,6 +587,8 @@ func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics {
}
return float64(val)
}),
m.walReplayUnknownRefsTotal,
m.wblReplayUnknownRefsTotal,
)
}
return m

20
tsdb/head_wal.go
View File

@ -27,6 +27,8 @@ import (
"go.uber.org/atomic"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/prometheus/model/exemplar"
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/model/labels"
@ -68,6 +70,12 @@ func (s *seriesRefSet) count() int {
return len(s.refs)
}
func counterAddNonZero(v *prometheus.CounterVec, value float64, lvs ...string) {
if value > 0 {
v.WithLabelValues(lvs...).Add(value)
}
}
func (h *Head) loadWAL(r *wlog.Reader, syms *labels.SymbolTable, multiRef map[chunks.HeadSeriesRef]chunks.HeadSeriesRef, mmappedChunks, oooMmappedChunks map[chunks.HeadSeriesRef][]*mmappedChunk, lastSegment int) (err error) {
// Track number of missing series records that were referenced by other records.
unknownSeriesRefs := &seriesRefSet{refs: make(map[chunks.HeadSeriesRef]struct{}), mtx: sync.Mutex{}}
@ -450,6 +458,13 @@ Outer:
"metadata", unknownMetadataRefs.Load(),
"tombstones", unknownTombstoneRefs.Load(),
)
counterAddNonZero(h.metrics.walReplayUnknownRefsTotal, float64(unknownSeriesRefs.count()), "series")
counterAddNonZero(h.metrics.walReplayUnknownRefsTotal, float64(unknownSampleRefs.Load()), "samples")
counterAddNonZero(h.metrics.walReplayUnknownRefsTotal, float64(unknownExemplarRefs.Load()), "exemplars")
counterAddNonZero(h.metrics.walReplayUnknownRefsTotal, float64(unknownHistogramRefs.Load()), "histograms")
counterAddNonZero(h.metrics.walReplayUnknownRefsTotal, float64(unknownMetadataRefs.Load()), "metadata")
counterAddNonZero(h.metrics.walReplayUnknownRefsTotal, float64(unknownTombstoneRefs.Load()), "tombstones")
}
if count := mmapOverlappingChunks.Load(); count > 0 {
h.logger.Info("Overlapping m-map chunks on duplicate series records", "count", count)
@ -932,6 +947,11 @@ func (h *Head) loadWBL(r *wlog.Reader, syms *labels.SymbolTable, multiRef map[ch
"histograms", unknownHistogramRefs.Load(),
"mmap_markers", mmapMarkerUnknownRefs.Load(),
)
counterAddNonZero(h.metrics.wblReplayUnknownRefsTotal, float64(unknownSeriesRefs.count()), "series")
counterAddNonZero(h.metrics.wblReplayUnknownRefsTotal, float64(unknownSampleRefs.Load()), "samples")
counterAddNonZero(h.metrics.wblReplayUnknownRefsTotal, float64(unknownHistogramRefs.Load()), "histograms")
counterAddNonZero(h.metrics.wblReplayUnknownRefsTotal, float64(mmapMarkerUnknownRefs.Load()), "mmap_markers")
}
return nil