mirror of
				https://github.com/prometheus/prometheus.git
				synced 2025-10-31 16:31:03 +01:00 
			
		
		
		
	* Introduce out-of-order TSDB support This implementation is based on this design doc: https://docs.google.com/document/d/1Kppm7qL9C-BJB1j6yb6-9ObG3AbdZnFUBYPNNWwDBYM/edit?usp=sharing This commit adds support to accept out-of-order ("OOO") sample into the TSDB up to a configurable time allowance. If OOO is enabled, overlapping querying are automatically enabled. Most of the additions have been borrowed from https://github.com/grafana/mimir-prometheus/ Here is the list ist of the original commits cherry picked from mimir-prometheus into this branch: - 4b2198d7ec47d50989b7c2df66b7b207c32f7f6e - 2836e5513f1bc591535a859f5d41154a75e7c6bc - 00b379c3a5b1ec3799699b6242f300a2b3ea30f0 - ff0dc757587cada63ca948d2d5eb00bf090d63e0 - a632c73352a7e39d60b445700beb47d691549c3e - c6f3d4ab339ab80bbbce74c9946237ced01f0509 - 5e8406a1d4a50d0052bbee83e28ca3b3371408aa - abde1e0ba128936b9eb0224ee1551e56216ebd4a - e70e7698897bb03860bee0467c733fa44e14c9bd - df59320886e03a555d379ac4b0b3130f661407e0 Co-authored-by: Jesus Vazquez <jesus.vazquez@grafana.com> Co-authored-by: Ganesh Vernekar <ganeshvern@gmail.com> Co-authored-by: Dieter Plaetinck <dieter@grafana.com> Signed-off-by: Jesus Vazquez <jesus.vazquez@grafana.com> * gofumpt files Signed-off-by: Jesus Vazquez <jesus.vazquez@grafana.com> * Add license header to missing files Signed-off-by: Jesus Vazquez <jesus.vazquez@grafana.com> * Fix OOO tests due to existing chunk disk mapper implementation Signed-off-by: Jesus Vazquez <jesus.vazquez@grafana.com> * Fix truncate int overflow Signed-off-by: Jesus Vazquez <jesus.vazquez@grafana.com> * Add Sync method to the WAL and update tests Signed-off-by: Jesus Vazquez <jesus.vazquez@grafana.com> * remove useless sync Signed-off-by: Jesus Vazquez <jesus.vazquez@grafana.com> * Update minOOOTime after truncating Head * Update minOOOTime after truncating Head Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com> * Fix lint Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com> * Add a unit test Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com> Signed-off-by: Jesus Vazquez <jesus.vazquez@grafana.com> * Load OutOfOrderTimeWindow only once per appender Signed-off-by: Jesus Vazquez <jesus.vazquez@grafana.com> * Fix OOO Head LabelValues and PostingsForMatchers Signed-off-by: Jesus Vazquez <jesus.vazquez@grafana.com> * Fix replay of OOO mmap chunks Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com> * Remove unnecessary err check Signed-off-by: Jesus Vazquez <jesus.vazquez@grafana.com> * Prevent panic with ApplyConfig Signed-off-by: Ganesh Vernekar 15064823+codesome@users.noreply.github.com Signed-off-by: Jesus Vazquez <jesus.vazquez@grafana.com> * Run OOO compaction after restart if there is OOO data from WBL Signed-off-by: Ganesh Vernekar 15064823+codesome@users.noreply.github.com Signed-off-by: Jesus Vazquez <jesus.vazquez@grafana.com> * Apply Bartek's suggestions Co-authored-by: Bartlomiej Plotka <bwplotka@gmail.com> Signed-off-by: Jesus Vazquez <jesus.vazquez@grafana.com> * Refactor OOO compaction Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com> * Address comments and TODOs - Added a comment explaining why we need the allow overlapping compaction toggle - Clarified TSDBConfig OutOfOrderTimeWindow doc - Added an owner to all the TODOs in the code Signed-off-by: Jesus Vazquez <jesus.vazquez@grafana.com> * Run go format Signed-off-by: Jesus Vazquez <jesus.vazquez@grafana.com> * Fix remaining review comments Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com> * Fix tests Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com> * Change wbl reference when truncating ooo in TestHeadMinOOOTimeUpdate Signed-off-by: Jesus Vazquez <jesus.vazquez@grafana.com> * Fix TestWBLAndMmapReplay test failure on windows Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com> * Address most of the feedback Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com> * Refactor the block meta for out of order Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com> * Fix windows error Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com> * Fix review comments Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com> Signed-off-by: Jesus Vazquez <jesus.vazquez@grafana.com> Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com> Signed-off-by: Ganesh Vernekar 15064823+codesome@users.noreply.github.com Co-authored-by: Ganesh Vernekar <15064823+codesome@users.noreply.github.com> Co-authored-by: Ganesh Vernekar <ganeshvern@gmail.com> Co-authored-by: Dieter Plaetinck <dieter@grafana.com> Co-authored-by: Oleg Zaytsev <mail@olegzaytsev.com> Co-authored-by: Bartlomiej Plotka <bwplotka@gmail.com>
		
			
				
	
	
		
			86 lines
		
	
	
		
			2.5 KiB
		
	
	
	
		
			Go
		
	
	
	
	
	
			
		
		
	
	
			86 lines
		
	
	
		
			2.5 KiB
		
	
	
	
		
			Go
		
	
	
	
	
	
| // Copyright 2018 The Prometheus Authors
 | |
| // Licensed under the Apache License, Version 2.0 (the "License");
 | |
| // you may not use this file except in compliance with the License.
 | |
| // You may obtain a copy of the License at
 | |
| //
 | |
| // http://www.apache.org/licenses/LICENSE-2.0
 | |
| //
 | |
| // Unless required by applicable law or agreed to in writing, software
 | |
| // distributed under the License is distributed on an "AS IS" BASIS,
 | |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 | |
| // See the License for the specific language governing permissions and
 | |
| // limitations under the License.
 | |
| 
 | |
| package tsdb
 | |
| 
 | |
| import (
 | |
| 	"strconv"
 | |
| 	"testing"
 | |
| 
 | |
| 	"github.com/pkg/errors"
 | |
| 	"github.com/stretchr/testify/require"
 | |
| 	"go.uber.org/atomic"
 | |
| 
 | |
| 	"github.com/prometheus/prometheus/model/labels"
 | |
| )
 | |
| 
 | |
| func BenchmarkHeadStripeSeriesCreate(b *testing.B) {
 | |
| 	chunkDir := b.TempDir()
 | |
| 	// Put a series, select it. GC it and then access it.
 | |
| 	opts := DefaultHeadOptions()
 | |
| 	opts.ChunkRange = 1000
 | |
| 	opts.ChunkDirRoot = chunkDir
 | |
| 	h, err := NewHead(nil, nil, nil, nil, opts, nil)
 | |
| 	require.NoError(b, err)
 | |
| 	defer h.Close()
 | |
| 
 | |
| 	for i := 0; i < b.N; i++ {
 | |
| 		h.getOrCreate(uint64(i), labels.FromStrings("a", strconv.Itoa(i)))
 | |
| 	}
 | |
| }
 | |
| 
 | |
| func BenchmarkHeadStripeSeriesCreateParallel(b *testing.B) {
 | |
| 	chunkDir := b.TempDir()
 | |
| 	// Put a series, select it. GC it and then access it.
 | |
| 	opts := DefaultHeadOptions()
 | |
| 	opts.ChunkRange = 1000
 | |
| 	opts.ChunkDirRoot = chunkDir
 | |
| 	h, err := NewHead(nil, nil, nil, nil, opts, nil)
 | |
| 	require.NoError(b, err)
 | |
| 	defer h.Close()
 | |
| 
 | |
| 	var count atomic.Int64
 | |
| 
 | |
| 	b.RunParallel(func(pb *testing.PB) {
 | |
| 		for pb.Next() {
 | |
| 			i := count.Inc()
 | |
| 			h.getOrCreate(uint64(i), labels.FromStrings("a", strconv.Itoa(int(i))))
 | |
| 		}
 | |
| 	})
 | |
| }
 | |
| 
 | |
| func BenchmarkHeadStripeSeriesCreate_PreCreationFailure(b *testing.B) {
 | |
| 	chunkDir := b.TempDir()
 | |
| 	// Put a series, select it. GC it and then access it.
 | |
| 	opts := DefaultHeadOptions()
 | |
| 	opts.ChunkRange = 1000
 | |
| 	opts.ChunkDirRoot = chunkDir
 | |
| 
 | |
| 	// Mock the PreCreation() callback to fail on each series.
 | |
| 	opts.SeriesCallback = failingSeriesLifecycleCallback{}
 | |
| 
 | |
| 	h, err := NewHead(nil, nil, nil, nil, opts, nil)
 | |
| 	require.NoError(b, err)
 | |
| 	defer h.Close()
 | |
| 
 | |
| 	for i := 0; i < b.N; i++ {
 | |
| 		h.getOrCreate(uint64(i), labels.FromStrings("a", strconv.Itoa(i)))
 | |
| 	}
 | |
| }
 | |
| 
 | |
| type failingSeriesLifecycleCallback struct{}
 | |
| 
 | |
| func (failingSeriesLifecycleCallback) PreCreation(labels.Labels) error { return errors.New("failed") }
 | |
| func (failingSeriesLifecycleCallback) PostCreation(labels.Labels)      {}
 | |
| func (failingSeriesLifecycleCallback) PostDeletion(...labels.Labels)   {}
 |