From d6258aea8f94bda501a227f37f940e32f96808cf Mon Sep 17 00:00:00 2001 From: Callum Styan Date: Wed, 13 Feb 2019 17:11:17 -0800 Subject: [PATCH] Fix up remote write tests: - Tests that created a QueueManager were leaving behind files at the end of tests. - WAL replaying (readToEnd)tests seem to require extra time to finish now. - Some fixes to make staticcheck happy Signed-off-by: Callum Styan --- storage/remote/queue_manager_test.go | 43 ++++++++++++++++++++++++---- storage/remote/wal_watcher.go | 2 +- storage/remote/wal_watcher_test.go | 14 +++------ 3 files changed, 42 insertions(+), 17 deletions(-) diff --git a/storage/remote/queue_manager_test.go b/storage/remote/queue_manager_test.go index 789875d6d8..e34ae048be 100644 --- a/storage/remote/queue_manager_test.go +++ b/storage/remote/queue_manager_test.go @@ -16,6 +16,8 @@ package remote import ( "context" "fmt" + "io/ioutil" + "os" "reflect" "sync" "sync/atomic" @@ -49,7 +51,12 @@ func TestSampleDelivery(t *testing.T) { cfg.BatchSendDeadline = model.Duration(100 * time.Millisecond) cfg.MaxShards = 1 var temp int64 - m := NewQueueManager(nil, "", newEWMARate(ewmaWeight, shardUpdateDuration), &temp, cfg, nil, nil, c, defaultFlushDeadline, 0) + + dir, err := ioutil.TempDir("", "TestSampleDeliver") + testutil.Ok(t, err) + defer os.RemoveAll(dir) + + m := NewQueueManager(nil, dir, newEWMARate(ewmaWeight, shardUpdateDuration), &temp, cfg, nil, nil, c, defaultFlushDeadline, 0) m.seriesLabels = refSeriesToLabelsProto(series) // These should be received by the client. @@ -73,7 +80,12 @@ func TestSampleDeliveryTimeout(t *testing.T) { cfg.MaxShards = 1 cfg.BatchSendDeadline = model.Duration(100 * time.Millisecond) var temp int64 - m := NewQueueManager(nil, "", newEWMARate(ewmaWeight, shardUpdateDuration), &temp, cfg, nil, nil, c, defaultFlushDeadline, 0) + + dir, err := ioutil.TempDir("", "TestSampleDeliveryTimeout") + testutil.Ok(t, err) + defer os.RemoveAll(dir) + + m := NewQueueManager(nil, dir, newEWMARate(ewmaWeight, shardUpdateDuration), &temp, cfg, nil, nil, c, defaultFlushDeadline, 0) m.seriesLabels = refSeriesToLabelsProto(series) m.Start() defer m.Stop() @@ -109,7 +121,12 @@ func TestSampleDeliveryOrder(t *testing.T) { c := NewTestStorageClient() c.expectSamples(samples, series) var temp int64 - m := NewQueueManager(nil, "", newEWMARate(ewmaWeight, shardUpdateDuration), &temp, config.DefaultQueueConfig, nil, nil, c, defaultFlushDeadline, 0) + + dir, err := ioutil.TempDir("", "TestSampleDeliveryOrder") + testutil.Ok(t, err) + defer os.RemoveAll(dir) + + m := NewQueueManager(nil, dir, newEWMARate(ewmaWeight, shardUpdateDuration), &temp, config.DefaultQueueConfig, nil, nil, c, defaultFlushDeadline, 0) m.seriesLabels = refSeriesToLabelsProto(series) m.Start() @@ -124,7 +141,12 @@ func TestShutdown(t *testing.T) { c := NewTestBlockedStorageClient() var temp int64 - m := NewQueueManager(nil, "", newEWMARate(ewmaWeight, shardUpdateDuration), &temp, config.DefaultQueueConfig, nil, nil, c, deadline, 0) + + dir, err := ioutil.TempDir("", "TestShutdown") + testutil.Ok(t, err) + defer os.RemoveAll(dir) + + m := NewQueueManager(nil, dir, newEWMARate(ewmaWeight, shardUpdateDuration), &temp, config.DefaultQueueConfig, nil, nil, c, deadline, 0) samples, series := createTimeseries(2 * config.DefaultQueueConfig.MaxSamplesPerSend) m.seriesLabels = refSeriesToLabelsProto(series) m.Start() @@ -157,7 +179,11 @@ func TestSeriesReset(t *testing.T) { numSegments := 4 numSeries := 25 - m := NewQueueManager(nil, "", newEWMARate(ewmaWeight, shardUpdateDuration), &temp, config.DefaultQueueConfig, nil, nil, c, deadline, 0) + dir, err := ioutil.TempDir("", "TestSeriesReset") + testutil.Ok(t, err) + defer os.RemoveAll(dir) + + m := NewQueueManager(nil, dir, newEWMARate(ewmaWeight, shardUpdateDuration), &temp, config.DefaultQueueConfig, nil, nil, c, deadline, 0) for i := 0; i < numSegments; i++ { series := []tsdb.RefSeries{} for j := 0; j < numSeries; j++ { @@ -182,7 +208,12 @@ func TestReshard(t *testing.T) { cfg.MaxShards = 1 var temp int64 - m := NewQueueManager(nil, "", newEWMARate(ewmaWeight, shardUpdateDuration), &temp, cfg, nil, nil, c, defaultFlushDeadline, 0) + + dir, err := ioutil.TempDir("", "TestReshard") + testutil.Ok(t, err) + defer os.RemoveAll(dir) + + m := NewQueueManager(nil, dir, newEWMARate(ewmaWeight, shardUpdateDuration), &temp, cfg, nil, nil, c, defaultFlushDeadline, 0) m.seriesLabels = refSeriesToLabelsProto(series) m.Start() diff --git a/storage/remote/wal_watcher.go b/storage/remote/wal_watcher.go index 4cdc37f5ee..b25c4efcf2 100644 --- a/storage/remote/wal_watcher.go +++ b/storage/remote/wal_watcher.go @@ -224,7 +224,7 @@ func (w *WALWatcher) findSegmentForIndex(index int) (int, error) { refs = append(refs, k) last = k } - sort.Sort(sort.IntSlice(refs)) + sort.Ints(refs) for _, r := range refs { if r >= index { diff --git a/storage/remote/wal_watcher_test.go b/storage/remote/wal_watcher_test.go index 9a68a5e3e3..8e009dce2e 100644 --- a/storage/remote/wal_watcher_test.go +++ b/storage/remote/wal_watcher_test.go @@ -372,20 +372,17 @@ func Test_decodeRecord(t *testing.T) { defer os.RemoveAll(dir) wt := newWriteToMock() - // st := timestamp.FromTime(time.Now().Add(-10 * time.Second)) watcher := NewWALWatcher(nil, "", wt, dir, 0) - // decode a series record enc := tsdb.RecordEncoder{} buf := enc.Series([]tsdb.RefSeries{tsdb.RefSeries{Ref: 1234, Labels: labels.Labels{}}}, nil) - watcher.decodeRecord(buf) + watcher.decodeRecord(buf, 0) testutil.Ok(t, err) testutil.Equals(t, 1, wt.checkNumLabels()) - // decode a samples record buf = enc.Samples([]tsdb.RefSample{tsdb.RefSample{Ref: 100, T: 1, V: 1.0}, tsdb.RefSample{Ref: 100, T: 2, V: 2.0}}, nil) - watcher.decodeRecord(buf) + watcher.decodeRecord(buf, 0) testutil.Ok(t, err) testutil.Equals(t, 2, wt.samplesAppended) @@ -397,20 +394,17 @@ func Test_decodeRecord_afterStart(t *testing.T) { defer os.RemoveAll(dir) wt := newWriteToMock() - // st := timestamp.FromTime(time.Now().Add(-10 * time.Second)) watcher := NewWALWatcher(nil, "", wt, dir, 1) - // decode a series record enc := tsdb.RecordEncoder{} buf := enc.Series([]tsdb.RefSeries{tsdb.RefSeries{Ref: 1234, Labels: labels.Labels{}}}, nil) - watcher.decodeRecord(buf) + watcher.decodeRecord(buf, 0) testutil.Ok(t, err) testutil.Equals(t, 1, wt.checkNumLabels()) - // decode a samples record buf = enc.Samples([]tsdb.RefSample{tsdb.RefSample{Ref: 100, T: 1, V: 1.0}, tsdb.RefSample{Ref: 100, T: 2, V: 2.0}}, nil) - watcher.decodeRecord(buf) + watcher.decodeRecord(buf, 0) testutil.Ok(t, err) testutil.Equals(t, 1, wt.samplesAppended)