mirrors/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2025-08-07 14:47:11 +02:00
Code Issues Projects Releases Wiki Activity

[BUGFIX] TSDB: Head compaction race condition

Browse Source 
There was a small time window where things can go wrong, if a query
starts after `WaitForPendingReadersInTimeRange` returns, but before
`minTime` is stored.

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
This commit is contained in:
Bryan Boreham 2024-10-04 16:25:27 -04:00
parent 29abf56799
commit a1f4a5c276
1 changed files with 7 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
tsdb/head.go
View File

@ -1165,18 +1165,21 @@ func (h *Head) truncateMemory(mint int64) (err error) {
h.memTruncationCallBack(1) h.memTruncationCallBack(1)
} }
oldMinTime := h.MinTime()
// Set the minTime that new queries will see, before waiting for queries to finish.
h.minTime.Store(mint)
// Don't allow samples to be added before this time.
h.minValidTime.Store(mint)
// We wait for pending queries to end that overlap with this truncation. // We wait for pending queries to end that overlap with this truncation.
if initialized { if initialized {
h.WaitForPendingReadersInTimeRange(h.MinTime(), mint) h.WaitForPendingReadersInTimeRange(oldMinTime, mint)
} }
if h.memTruncationCallBack != nil { if h.memTruncationCallBack != nil {
h.memTruncationCallBack(2) h.memTruncationCallBack(2)
} }
h.minTime.Store(mint)
h.minValidTime.Store(mint)
// Ensure that max time is at least as high as min time. // Ensure that max time is at least as high as min time.
for h.MaxTime() < mint { for h.MaxTime() < mint {
h.maxTime.CompareAndSwap(h.MaxTime(), mint) h.maxTime.CompareAndSwap(h.MaxTime(), mint)