diff --git a/prompb/io/prometheus/write/v2/codec.go b/prompb/io/prometheus/write/v2/codec.go index 25fa0d4035..4434c525fc 100644 --- a/prompb/io/prometheus/write/v2/codec.go +++ b/prompb/io/prometheus/write/v2/codec.go @@ -196,6 +196,9 @@ func FromFloatHistogram(timestamp int64, fh *histogram.FloatHistogram) Histogram } func spansToSpansProto(s []histogram.Span) []BucketSpan { + if len(s) == 0 { + return nil + } spans := make([]BucketSpan, len(s)) for i := 0; i < len(s); i++ { spans[i] = BucketSpan{Offset: s[i].Offset, Length: s[i].Length} diff --git a/storage/remote/codec_test.go b/storage/remote/codec_test.go index 51cdd1e39a..c92f0f8cde 100644 --- a/storage/remote/codec_test.go +++ b/storage/remote/codec_test.go @@ -43,12 +43,12 @@ var ( Schema: 2, ZeroThreshold: 1e-128, ZeroCount: 0, - Count: 0, + Count: 3, Sum: 20, PositiveSpans: []histogram.Span{{Offset: 0, Length: 1}}, PositiveBuckets: []int64{1}, NegativeSpans: []histogram.Span{{Offset: 0, Length: 1}}, - NegativeBuckets: []int64{-1}, + NegativeBuckets: []int64{2}, } writeRequestFixture = &prompb.WriteRequest{ @@ -90,6 +90,15 @@ var ( Help: "Test counter for test purposes", } + testHistogramCustomBuckets = histogram.Histogram{ + Schema: histogram.CustomBucketsSchema, + Count: 16, + Sum: 20, + PositiveSpans: []histogram.Span{{Offset: 1, Length: 2}}, + PositiveBuckets: []int64{10, -4}, // Means 10 observations for upper bound 1.0 and 6 for upper bound +Inf. + CustomValues: []float64{0.1, 1.0}, // +Inf is implied. + } + // writeV2RequestFixture represents the same request as writeRequestFixture, // but using the v2 representation, plus includes writeV2RequestSeries1Metadata and writeV2RequestSeries2Metadata. // NOTE: Use TestWriteV2RequestFixture and copy the diff to regenerate if needed. @@ -104,9 +113,14 @@ var ( HelpRef: 15, // Symbolized writeV2RequestSeries1Metadata.Help. UnitRef: 16, // Symbolized writeV2RequestSeries1Metadata.Unit. }, - Samples: []writev2.Sample{{Value: 1, Timestamp: 10}}, - Exemplars: []writev2.Exemplar{{LabelsRefs: []uint32{11, 12}, Value: 1, Timestamp: 10}}, - Histograms: []writev2.Histogram{writev2.FromIntHistogram(10, &testHistogram), writev2.FromFloatHistogram(20, testHistogram.ToFloat(nil))}, + Samples: []writev2.Sample{{Value: 1, Timestamp: 10}}, + Exemplars: []writev2.Exemplar{{LabelsRefs: []uint32{11, 12}, Value: 1, Timestamp: 10}}, + Histograms: []writev2.Histogram{ + writev2.FromIntHistogram(10, &testHistogram), + writev2.FromFloatHistogram(20, testHistogram.ToFloat(nil)), + writev2.FromIntHistogram(30, &testHistogramCustomBuckets), + writev2.FromFloatHistogram(40, testHistogramCustomBuckets.ToFloat(nil)), + }, CreatedTimestamp: 1, // CT needs to be lower than the sample's timestamp. }, { @@ -117,14 +131,40 @@ var ( HelpRef: 17, // Symbolized writeV2RequestSeries2Metadata.Help. // No unit. }, - Samples: []writev2.Sample{{Value: 2, Timestamp: 20}}, - Exemplars: []writev2.Exemplar{{LabelsRefs: []uint32{13, 14}, Value: 2, Timestamp: 20}}, - Histograms: []writev2.Histogram{writev2.FromIntHistogram(30, &testHistogram), writev2.FromFloatHistogram(40, testHistogram.ToFloat(nil))}, + Samples: []writev2.Sample{{Value: 2, Timestamp: 20}}, + Exemplars: []writev2.Exemplar{{LabelsRefs: []uint32{13, 14}, Value: 2, Timestamp: 20}}, + Histograms: []writev2.Histogram{ + writev2.FromIntHistogram(50, &testHistogram), + writev2.FromFloatHistogram(60, testHistogram.ToFloat(nil)), + writev2.FromIntHistogram(70, &testHistogramCustomBuckets), + writev2.FromFloatHistogram(80, testHistogramCustomBuckets.ToFloat(nil)), + }, }, }, } ) +func TestHistogramFixtureValid(t *testing.T) { + for _, ts := range writeRequestFixture.Timeseries { + for _, h := range ts.Histograms { + if h.IsFloatHistogram() { + require.NoError(t, h.ToFloatHistogram().Validate()) + } else { + require.NoError(t, h.ToIntHistogram().Validate()) + } + } + } + for _, ts := range writeV2RequestFixture.Timeseries { + for _, h := range ts.Histograms { + if h.IsFloatHistogram() { + require.NoError(t, h.ToFloatHistogram().Validate()) + } else { + require.NoError(t, h.ToIntHistogram().Validate()) + } + } + } +} + func TestWriteV2RequestFixture(t *testing.T) { // Generate dynamically writeV2RequestFixture, reusing v1 fixture elements. st := writev2.NewSymbolTable() @@ -141,9 +181,14 @@ func TestWriteV2RequestFixture(t *testing.T) { HelpRef: st.Symbolize(writeV2RequestSeries1Metadata.Help), UnitRef: st.Symbolize(writeV2RequestSeries1Metadata.Unit), }, - Samples: []writev2.Sample{{Value: 1, Timestamp: 10}}, - Exemplars: []writev2.Exemplar{{LabelsRefs: exemplar1LabelRefs, Value: 1, Timestamp: 10}}, - Histograms: []writev2.Histogram{writev2.FromIntHistogram(10, &testHistogram), writev2.FromFloatHistogram(20, testHistogram.ToFloat(nil))}, + Samples: []writev2.Sample{{Value: 1, Timestamp: 10}}, + Exemplars: []writev2.Exemplar{{LabelsRefs: exemplar1LabelRefs, Value: 1, Timestamp: 10}}, + Histograms: []writev2.Histogram{ + writev2.FromIntHistogram(10, &testHistogram), + writev2.FromFloatHistogram(20, testHistogram.ToFloat(nil)), + writev2.FromIntHistogram(30, &testHistogramCustomBuckets), + writev2.FromFloatHistogram(40, testHistogramCustomBuckets.ToFloat(nil)), + }, CreatedTimestamp: 1, }, { @@ -153,9 +198,14 @@ func TestWriteV2RequestFixture(t *testing.T) { HelpRef: st.Symbolize(writeV2RequestSeries2Metadata.Help), // No unit. }, - Samples: []writev2.Sample{{Value: 2, Timestamp: 20}}, - Exemplars: []writev2.Exemplar{{LabelsRefs: exemplar2LabelRefs, Value: 2, Timestamp: 20}}, - Histograms: []writev2.Histogram{writev2.FromIntHistogram(30, &testHistogram), writev2.FromFloatHistogram(40, testHistogram.ToFloat(nil))}, + Samples: []writev2.Sample{{Value: 2, Timestamp: 20}}, + Exemplars: []writev2.Exemplar{{LabelsRefs: exemplar2LabelRefs, Value: 2, Timestamp: 20}}, + Histograms: []writev2.Histogram{ + writev2.FromIntHistogram(50, &testHistogram), + writev2.FromFloatHistogram(60, testHistogram.ToFloat(nil)), + writev2.FromIntHistogram(70, &testHistogramCustomBuckets), + writev2.FromFloatHistogram(80, testHistogramCustomBuckets.ToFloat(nil)), + }, }, }, Symbols: st.Symbols(), diff --git a/storage/remote/write_handler_test.go b/storage/remote/write_handler_test.go index e92d24f66e..e3207bf273 100644 --- a/storage/remote/write_handler_test.go +++ b/storage/remote/write_handler_test.go @@ -391,7 +391,7 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) { desc: "Partial write; first series with one dup histogram sample", input: func() []writev2.TimeSeries { f := proto.Clone(writeV2RequestFixture).(*writev2.Request) - f.Timeseries[0].Histograms = append(f.Timeseries[0].Histograms, f.Timeseries[0].Histograms[1]) + f.Timeseries[0].Histograms = append(f.Timeseries[0].Histograms, f.Timeseries[0].Histograms[len(f.Timeseries[0].Histograms)-1]) return f.Timeseries }(), expectedCode: http.StatusBadRequest, @@ -483,7 +483,7 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) { // Double check mandatory 2.0 stats. // writeV2RequestFixture has 2 series with 1 sample, 2 histograms, 1 exemplar each. expectHeaderValue(t, 2, resp.Header.Get(rw20WrittenSamplesHeader)) - expectHeaderValue(t, 4, resp.Header.Get(rw20WrittenHistogramsHeader)) + expectHeaderValue(t, 8, resp.Header.Get(rw20WrittenHistogramsHeader)) if tc.appendExemplarErr != nil { expectHeaderValue(t, 0, resp.Header.Get(rw20WrittenExemplarsHeader)) } else { @@ -496,6 +496,8 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) { i, j, k, m int ) for _, ts := range writeV2RequestFixture.Timeseries { + zeroHistogramIngested := false + zeroFloatHistogramIngested := false ls := ts.ToLabels(&b, writeV2RequestFixture.Symbols) for _, s := range ts.Samples { @@ -509,21 +511,27 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) { for _, hp := range ts.Histograms { if hp.IsFloatHistogram() { fh := hp.ToFloatHistogram() - if ts.CreatedTimestamp != 0 && tc.ingestCTZeroSample { + if !zeroFloatHistogramIngested && ts.CreatedTimestamp != 0 && tc.ingestCTZeroSample { requireEqual(t, mockHistogram{ls, ts.CreatedTimestamp, nil, &histogram.FloatHistogram{}}, appendable.histograms[k]) k++ + zeroFloatHistogramIngested = true } requireEqual(t, mockHistogram{ls, hp.Timestamp, nil, fh}, appendable.histograms[k]) } else { h := hp.ToIntHistogram() - if ts.CreatedTimestamp != 0 && tc.ingestCTZeroSample { + if !zeroHistogramIngested && ts.CreatedTimestamp != 0 && tc.ingestCTZeroSample { requireEqual(t, mockHistogram{ls, ts.CreatedTimestamp, &histogram.Histogram{}, nil}, appendable.histograms[k]) k++ + zeroHistogramIngested = true } requireEqual(t, mockHistogram{ls, hp.Timestamp, h, nil}, appendable.histograms[k]) } k++ } + if ts.CreatedTimestamp != 0 && tc.ingestCTZeroSample { + require.True(t, zeroHistogramIngested) + require.True(t, zeroFloatHistogramIngested) + } if tc.appendExemplarErr == nil { for _, e := range ts.Exemplars { exemplarLabels := e.ToExemplar(&b, writeV2RequestFixture.Symbols).Labels