From 04aabdd7cc86fedba3dd2e614fa5289b746bda28 Mon Sep 17 00:00:00 2001 From: Jeanette Tan Date: Wed, 1 Nov 2023 19:53:41 +0800 Subject: [PATCH] Refactor TestPopulateWithDelSeriesIterator unit tests to reuse more code Signed-off-by: Jeanette Tan --- tsdb/querier_test.go | 385 +++++++++++++++++++++---------------------- 1 file changed, 189 insertions(+), 196 deletions(-) diff --git a/tsdb/querier_test.go b/tsdb/querier_test.go index 096e0ff729..1c50724d7c 100644 --- a/tsdb/querier_test.go +++ b/tsdb/querier_test.go @@ -1158,220 +1158,213 @@ func rmChunkRefs(chks []chunks.Meta) { } } +func checkCurrVal(t *testing.T, valType chunkenc.ValueType, it *populateWithDelSeriesIterator, expectedTs, expectedValue int) { + switch valType { + case chunkenc.ValFloat: + ts, v := it.At() + require.Equal(t, int64(expectedTs), ts) + require.Equal(t, float64(expectedValue), v) + case chunkenc.ValHistogram: + ts, h := it.AtHistogram() + require.Equal(t, int64(expectedTs), ts) + h.CounterResetHint = histogram.UnknownCounterReset + require.Equal(t, tsdbutil.GenerateTestHistogram(expectedValue), h) + case chunkenc.ValFloatHistogram: + ts, h := it.AtFloatHistogram() + require.Equal(t, int64(expectedTs), ts) + h.CounterResetHint = histogram.UnknownCounterReset + require.Equal(t, tsdbutil.GenerateTestFloatHistogram(expectedValue), h) + default: + panic("unexpected value type") + } +} + // Regression for: https://github.com/prometheus/tsdb/pull/97 func TestPopulateWithDelSeriesIterator_DoubleSeek(t *testing.T) { - t.Run("float", func(t *testing.T) { - valType := chunkenc.ValFloat - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{}, - []chunks.Sample{sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}}, - []chunks.Sample{sample{4, 4, nil, nil}, sample{5, 5, nil, nil}}, - ) - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, valType, it.Seek(1)) - require.Equal(t, valType, it.Seek(2)) - require.Equal(t, valType, it.Seek(2)) - ts, v := it.At() - require.Equal(t, int64(2), ts) - require.Equal(t, float64(2), v) - require.Equal(t, int64(0), chkMetas[0].MinTime) - require.Equal(t, int64(1), chkMetas[1].MinTime) - require.Equal(t, int64(4), chkMetas[2].MinTime) - }) - t.Run("histogram", func(t *testing.T) { - valType := chunkenc.ValHistogram - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{}, - []chunks.Sample{sample{1, 0, tsdbutil.GenerateTestHistogram(1), nil}, sample{2, 0, tsdbutil.GenerateTestHistogram(2), nil}, sample{3, 0, tsdbutil.GenerateTestHistogram(3), nil}}, - []chunks.Sample{sample{4, 0, tsdbutil.GenerateTestHistogram(4), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(5), nil}}, - ) - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, valType, it.Seek(1)) - require.Equal(t, valType, it.Seek(2)) - require.Equal(t, valType, it.Seek(2)) - ts, h := it.AtHistogram() - require.Equal(t, int64(2), ts) - h.CounterResetHint = histogram.UnknownCounterReset - require.Equal(t, tsdbutil.GenerateTestHistogram(2), h) - require.Equal(t, int64(0), chkMetas[0].MinTime) - require.Equal(t, int64(1), chkMetas[1].MinTime) - require.Equal(t, int64(4), chkMetas[2].MinTime) - }) - t.Run("float histogram", func(t *testing.T) { - valType := chunkenc.ValFloatHistogram - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{}, - []chunks.Sample{sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(1)}, sample{2, 0, nil, tsdbutil.GenerateTestFloatHistogram(2)}, sample{3, 0, nil, tsdbutil.GenerateTestFloatHistogram(3)}}, - []chunks.Sample{sample{4, 0, nil, tsdbutil.GenerateTestFloatHistogram(4)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(5)}}, - ) - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, valType, it.Seek(1)) - require.Equal(t, valType, it.Seek(2)) - require.Equal(t, valType, it.Seek(2)) - ts, h := it.AtFloatHistogram() - require.Equal(t, int64(2), ts) - h.CounterResetHint = histogram.UnknownCounterReset - require.Equal(t, tsdbutil.GenerateTestFloatHistogram(2), h) - require.Equal(t, int64(0), chkMetas[0].MinTime) - require.Equal(t, int64(1), chkMetas[1].MinTime) - require.Equal(t, int64(4), chkMetas[2].MinTime) - }) + cases := []struct { + name string + valType chunkenc.ValueType + chks [][]chunks.Sample + }{ + { + name: "float", + valType: chunkenc.ValFloat, + chks: [][]chunks.Sample{ + {}, + {sample{1, 1, nil, nil}, sample{2, 2, nil, nil}, sample{3, 3, nil, nil}}, + {sample{4, 4, nil, nil}, sample{5, 5, nil, nil}}, + }, + }, + { + name: "histogram", + valType: chunkenc.ValHistogram, + chks: [][]chunks.Sample{ + {}, + {sample{1, 0, tsdbutil.GenerateTestHistogram(1), nil}, sample{2, 0, tsdbutil.GenerateTestHistogram(2), nil}, sample{3, 0, tsdbutil.GenerateTestHistogram(3), nil}}, + {sample{4, 0, tsdbutil.GenerateTestHistogram(4), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(5), nil}}, + }, + }, + { + name: "float histogram", + valType: chunkenc.ValFloatHistogram, + chks: [][]chunks.Sample{ + {}, + {sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(1)}, sample{2, 0, nil, tsdbutil.GenerateTestFloatHistogram(2)}, sample{3, 0, nil, tsdbutil.GenerateTestFloatHistogram(3)}}, + {sample{4, 0, nil, tsdbutil.GenerateTestFloatHistogram(4)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(5)}}, + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f, chkMetas := createFakeReaderAndNotPopulatedChunks(tc.chks...) + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, tc.valType, it.Seek(1)) + require.Equal(t, tc.valType, it.Seek(2)) + require.Equal(t, tc.valType, it.Seek(2)) + checkCurrVal(t, tc.valType, it, 2, 2) + require.Equal(t, int64(0), chkMetas[0].MinTime) + require.Equal(t, int64(1), chkMetas[1].MinTime) + require.Equal(t, int64(4), chkMetas[2].MinTime) + }) + } } // Regression when seeked chunks were still found via binary search and we always // skipped to the end when seeking a value in the current chunk. func TestPopulateWithDelSeriesIterator_SeekInCurrentChunk(t *testing.T) { - t.Run("float", func(t *testing.T) { - valType := chunkenc.ValFloat - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{}, - []chunks.Sample{sample{1, 2, nil, nil}, sample{3, 4, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, - []chunks.Sample{}, - ) + cases := []struct { + name string + valType chunkenc.ValueType + chks [][]chunks.Sample + }{ + { + name: "float", + valType: chunkenc.ValFloat, + chks: [][]chunks.Sample{ + {}, + {sample{1, 2, nil, nil}, sample{3, 4, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, + {}, + }, + }, + { + name: "histogram", + valType: chunkenc.ValHistogram, + chks: [][]chunks.Sample{ + {}, + {sample{1, 0, tsdbutil.GenerateTestHistogram(2), nil}, sample{3, 0, tsdbutil.GenerateTestHistogram(4), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{7, 0, tsdbutil.GenerateTestHistogram(8), nil}}, + {}, + }, + }, + { + name: "float histogram", + valType: chunkenc.ValFloatHistogram, + chks: [][]chunks.Sample{ + {}, + {sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(2)}, sample{3, 0, nil, tsdbutil.GenerateTestFloatHistogram(4)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{7, 0, nil, tsdbutil.GenerateTestFloatHistogram(8)}}, + {}, + }, + }, + } - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, valType, it.Next()) - ts, v := it.At() - require.Equal(t, int64(1), ts) - require.Equal(t, float64(2), v) - - require.Equal(t, valType, it.Seek(4)) - ts, v = it.At() - require.Equal(t, int64(5), ts) - require.Equal(t, float64(6), v) - - require.Equal(t, int64(0), chkMetas[0].MinTime) - require.Equal(t, int64(1), chkMetas[1].MinTime) - require.Equal(t, int64(0), chkMetas[2].MinTime) - }) - t.Run("histogram", func(t *testing.T) { - valType := chunkenc.ValHistogram - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{}, - []chunks.Sample{sample{1, 0, tsdbutil.GenerateTestHistogram(2), nil}, sample{3, 0, tsdbutil.GenerateTestHistogram(4), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{7, 0, tsdbutil.GenerateTestHistogram(8), nil}}, - []chunks.Sample{}, - ) - - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, valType, it.Next()) - ts, h := it.AtHistogram() - require.Equal(t, int64(1), ts) - require.Equal(t, tsdbutil.GenerateTestHistogram(2), h) - - require.Equal(t, valType, it.Seek(4)) - ts, h = it.AtHistogram() - require.Equal(t, int64(5), ts) - h.CounterResetHint = histogram.UnknownCounterReset - require.Equal(t, tsdbutil.GenerateTestHistogram(6), h) - - require.Equal(t, int64(0), chkMetas[0].MinTime) - require.Equal(t, int64(1), chkMetas[1].MinTime) - require.Equal(t, int64(0), chkMetas[2].MinTime) - }) - t.Run("float histogram", func(t *testing.T) { - valType := chunkenc.ValFloatHistogram - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{}, - []chunks.Sample{sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(2)}, sample{3, 0, nil, tsdbutil.GenerateTestFloatHistogram(4)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{7, 0, nil, tsdbutil.GenerateTestFloatHistogram(8)}}, - []chunks.Sample{}, - ) - - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, valType, it.Next()) - ts, h := it.AtFloatHistogram() - require.Equal(t, int64(1), ts) - require.Equal(t, tsdbutil.GenerateTestFloatHistogram(2), h) - - require.Equal(t, valType, it.Seek(4)) - ts, h = it.AtFloatHistogram() - require.Equal(t, int64(5), ts) - h.CounterResetHint = histogram.UnknownCounterReset - require.Equal(t, tsdbutil.GenerateTestFloatHistogram(6), h) - - require.Equal(t, int64(0), chkMetas[0].MinTime) - require.Equal(t, int64(1), chkMetas[1].MinTime) - require.Equal(t, int64(0), chkMetas[2].MinTime) - }) + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f, chkMetas := createFakeReaderAndNotPopulatedChunks(tc.chks...) + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, tc.valType, it.Next()) + checkCurrVal(t, tc.valType, it, 1, 2) + require.Equal(t, tc.valType, it.Seek(4)) + checkCurrVal(t, tc.valType, it, 5, 6) + require.Equal(t, int64(0), chkMetas[0].MinTime) + require.Equal(t, int64(1), chkMetas[1].MinTime) + require.Equal(t, int64(0), chkMetas[2].MinTime) + }) + } } func TestPopulateWithDelSeriesIterator_SeekWithMinTime(t *testing.T) { - t.Run("float", func(t *testing.T) { - valType := chunkenc.ValFloat - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{6, 8, nil, nil}}, - ) + cases := []struct { + name string + valType chunkenc.ValueType + chks [][]chunks.Sample + }{ + { + name: "float", + valType: chunkenc.ValFloat, + chks: [][]chunks.Sample{ + {sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{6, 8, nil, nil}}, + }, + }, + { + name: "histogram", + valType: chunkenc.ValHistogram, + chks: [][]chunks.Sample{ + {sample{1, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{6, 0, tsdbutil.GenerateTestHistogram(8), nil}}, + }, + }, + { + name: "float histogram", + valType: chunkenc.ValFloatHistogram, + chks: [][]chunks.Sample{ + {sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{6, 0, nil, tsdbutil.GenerateTestFloatHistogram(8)}}, + }, + }, + } - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, chunkenc.ValNone, it.Seek(7)) - require.Equal(t, valType, it.Seek(3)) - require.Equal(t, int64(1), chkMetas[0].MinTime) - }) - t.Run("histogram", func(t *testing.T) { - valType := chunkenc.ValHistogram - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{sample{1, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{6, 0, tsdbutil.GenerateTestHistogram(8), nil}}, - ) - - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, chunkenc.ValNone, it.Seek(7)) - require.Equal(t, valType, it.Seek(3)) - require.Equal(t, int64(1), chkMetas[0].MinTime) - }) - t.Run("float histogram", func(t *testing.T) { - valType := chunkenc.ValFloatHistogram - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{6, 0, nil, tsdbutil.GenerateTestFloatHistogram(8)}}, - ) - - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, nil) - require.Equal(t, chunkenc.ValNone, it.Seek(7)) - require.Equal(t, valType, it.Seek(3)) - require.Equal(t, int64(1), chkMetas[0].MinTime) - }) + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f, chkMetas := createFakeReaderAndNotPopulatedChunks(tc.chks...) + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, nil) + require.Equal(t, chunkenc.ValNone, it.Seek(7)) + require.Equal(t, tc.valType, it.Seek(3)) + require.Equal(t, int64(1), chkMetas[0].MinTime) + }) + } } // Regression when calling Next() with a time bounded to fit within two samples. // Seek gets called and advances beyond the max time, which was just accepted as a valid sample. func TestPopulateWithDelSeriesIterator_NextWithMinTime(t *testing.T) { - t.Run("float", func(t *testing.T) { - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, - ) + cases := []struct { + name string + valType chunkenc.ValueType + chks [][]chunks.Sample + }{ + { + name: "float", + valType: chunkenc.ValFloat, + chks: [][]chunks.Sample{ + {sample{1, 6, nil, nil}, sample{5, 6, nil, nil}, sample{7, 8, nil, nil}}, + }, + }, + { + name: "histogram", + valType: chunkenc.ValHistogram, + chks: [][]chunks.Sample{ + {sample{1, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{7, 0, tsdbutil.GenerateTestHistogram(8), nil}}, + }, + }, + { + name: "float histogram", + valType: chunkenc.ValFloatHistogram, + chks: [][]chunks.Sample{ + {sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{7, 0, nil, tsdbutil.GenerateTestFloatHistogram(8)}}, + }, + }, + } - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, tombstones.Intervals{{Mint: math.MinInt64, Maxt: 2}}.Add(tombstones.Interval{Mint: 4, Maxt: math.MaxInt64})) - require.Equal(t, chunkenc.ValNone, it.Next()) - require.Equal(t, int64(1), chkMetas[0].MinTime) - }) - t.Run("histogram", func(t *testing.T) { - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{sample{1, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{5, 0, tsdbutil.GenerateTestHistogram(6), nil}, sample{7, 0, tsdbutil.GenerateTestHistogram(8), nil}}, - ) - - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, tombstones.Intervals{{Mint: math.MinInt64, Maxt: 2}}.Add(tombstones.Interval{Mint: 4, Maxt: math.MaxInt64})) - require.Equal(t, chunkenc.ValNone, it.Next()) - require.Equal(t, int64(1), chkMetas[0].MinTime) - }) - t.Run("float histogram", func(t *testing.T) { - f, chkMetas := createFakeReaderAndNotPopulatedChunks( - []chunks.Sample{sample{1, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{5, 0, nil, tsdbutil.GenerateTestFloatHistogram(6)}, sample{7, 0, nil, tsdbutil.GenerateTestFloatHistogram(8)}}, - ) - - it := &populateWithDelSeriesIterator{} - it.reset(ulid.ULID{}, f, chkMetas, tombstones.Intervals{{Mint: math.MinInt64, Maxt: 2}}.Add(tombstones.Interval{Mint: 4, Maxt: math.MaxInt64})) - require.Equal(t, chunkenc.ValNone, it.Next()) - require.Equal(t, int64(1), chkMetas[0].MinTime) - }) + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f, chkMetas := createFakeReaderAndNotPopulatedChunks(tc.chks...) + it := &populateWithDelSeriesIterator{} + it.reset(ulid.ULID{}, f, chkMetas, tombstones.Intervals{{Mint: math.MinInt64, Maxt: 2}}.Add(tombstones.Interval{Mint: 4, Maxt: math.MaxInt64})) + require.Equal(t, chunkenc.ValNone, it.Next()) + require.Equal(t, int64(1), chkMetas[0].MinTime) + }) + } } // Test the cost of merging series sets for different number of merged sets and their size.