diff --git a/tsdb/head.go b/tsdb/head.go index 4f4e4febfc..724437f453 100644 --- a/tsdb/head.go +++ b/tsdb/head.go @@ -379,6 +379,8 @@ type headMetrics struct { snapshotReplayErrorTotal prometheus.Counter // Will be either 0 or 1. oooHistogram prometheus.Histogram mmapChunksTotal prometheus.Counter + walReplayUnknownRefsTotal *prometheus.CounterVec + wblReplayUnknownRefsTotal *prometheus.CounterVec } const ( @@ -510,6 +512,14 @@ func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics { Name: "prometheus_tsdb_mmap_chunks_total", Help: "Total number of chunks that were memory-mapped.", }), + walReplayUnknownRefsTotal: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "prometheus_tsdb_wal_replay_unknown_refs_total", + Help: "Total number of unknown series references encountered during WAL replay.", + }, []string{"type"}), + wblReplayUnknownRefsTotal: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "prometheus_tsdb_wbl_replay_unknown_refs_total", + Help: "Total number of unknown series references encountered during WBL replay.", + }, []string{"type"}), } if r != nil { @@ -577,6 +587,8 @@ func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics { } return float64(val) }), + m.walReplayUnknownRefsTotal, + m.wblReplayUnknownRefsTotal, ) } return m diff --git a/tsdb/head_wal.go b/tsdb/head_wal.go index 4fdb1bb8e4..4de091c291 100644 --- a/tsdb/head_wal.go +++ b/tsdb/head_wal.go @@ -27,6 +27,8 @@ import ( "go.uber.org/atomic" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/model/exemplar" "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" @@ -68,6 +70,12 @@ func (s *seriesRefSet) count() int { return len(s.refs) } +func counterAddNonZero(v *prometheus.CounterVec, value float64, lvs ...string) { + if value > 0 { + v.WithLabelValues(lvs...).Add(value) + } +} + func (h *Head) loadWAL(r *wlog.Reader, syms *labels.SymbolTable, multiRef map[chunks.HeadSeriesRef]chunks.HeadSeriesRef, mmappedChunks, oooMmappedChunks map[chunks.HeadSeriesRef][]*mmappedChunk, lastSegment int) (err error) { // Track number of missing series records that were referenced by other records. unknownSeriesRefs := &seriesRefSet{refs: make(map[chunks.HeadSeriesRef]struct{}), mtx: sync.Mutex{}} @@ -450,6 +458,13 @@ Outer: "metadata", unknownMetadataRefs.Load(), "tombstones", unknownTombstoneRefs.Load(), ) + + counterAddNonZero(h.metrics.walReplayUnknownRefsTotal, float64(unknownSeriesRefs.count()), "series") + counterAddNonZero(h.metrics.walReplayUnknownRefsTotal, float64(unknownSampleRefs.Load()), "samples") + counterAddNonZero(h.metrics.walReplayUnknownRefsTotal, float64(unknownExemplarRefs.Load()), "exemplars") + counterAddNonZero(h.metrics.walReplayUnknownRefsTotal, float64(unknownHistogramRefs.Load()), "histograms") + counterAddNonZero(h.metrics.walReplayUnknownRefsTotal, float64(unknownMetadataRefs.Load()), "metadata") + counterAddNonZero(h.metrics.walReplayUnknownRefsTotal, float64(unknownTombstoneRefs.Load()), "tombstones") } if count := mmapOverlappingChunks.Load(); count > 0 { h.logger.Info("Overlapping m-map chunks on duplicate series records", "count", count) @@ -932,6 +947,11 @@ func (h *Head) loadWBL(r *wlog.Reader, syms *labels.SymbolTable, multiRef map[ch "histograms", unknownHistogramRefs.Load(), "mmap_markers", mmapMarkerUnknownRefs.Load(), ) + + counterAddNonZero(h.metrics.wblReplayUnknownRefsTotal, float64(unknownSeriesRefs.count()), "series") + counterAddNonZero(h.metrics.wblReplayUnknownRefsTotal, float64(unknownSampleRefs.Load()), "samples") + counterAddNonZero(h.metrics.wblReplayUnknownRefsTotal, float64(unknownHistogramRefs.Load()), "histograms") + counterAddNonZero(h.metrics.wblReplayUnknownRefsTotal, float64(mmapMarkerUnknownRefs.Load()), "mmap_markers") } return nil