From 686dcc7b0d78920553de5181689464cbc3e57dee Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Fri, 7 Feb 2025 16:46:55 +0100 Subject: [PATCH] headIndexReader: reduce debug logging (#15993) Around Mimir compactions we see logging in ShardedPostings do massive allocations and drive GC up to 50% of CPU. Signed-off-by: Dimitar Dimitrov --- tsdb/head_read.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tsdb/head_read.go b/tsdb/head_read.go index b95257c28a..438c7e6a4d 100644 --- a/tsdb/head_read.go +++ b/tsdb/head_read.go @@ -117,15 +117,19 @@ func (h *headIndexReader) PostingsForAllLabelValues(ctx context.Context, name st func (h *headIndexReader) SortedPostings(p index.Postings) index.Postings { series := make([]*memSeries, 0, 128) + notFoundSeriesCount := 0 // Fetch all the series only once. for p.Next() { s := h.head.series.getByID(chunks.HeadSeriesRef(p.At())) if s == nil { - h.head.logger.Debug("Looked up series not found") + notFoundSeriesCount++ } else { series = append(series, s) } } + if notFoundSeriesCount > 0 { + h.head.logger.Debug("Looked up series not found", "count", notFoundSeriesCount) + } if err := p.Err(); err != nil { return index.ErrPostings(fmt.Errorf("expand postings: %w", err)) } @@ -150,11 +154,12 @@ func (h *headIndexReader) ShardedPostings(p index.Postings, shardIndex, shardCou } out := make([]storage.SeriesRef, 0, 128) + notFoundSeriesCount := 0 for p.Next() { s := h.head.series.getByID(chunks.HeadSeriesRef(p.At())) if s == nil { - h.head.logger.Debug("Looked up series not found") + notFoundSeriesCount++ continue } @@ -165,6 +170,9 @@ func (h *headIndexReader) ShardedPostings(p index.Postings, shardIndex, shardCou out = append(out, storage.SeriesRef(s.ref)) } + if notFoundSeriesCount > 0 { + h.head.logger.Debug("Looked up series not found", "count", notFoundSeriesCount) + } return index.NewListPostings(out) }