mirrors/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2025-08-07 14:47:11 +02:00
Code Issues Projects Releases Wiki Activity

Merge pull request #16409 from prometheus/krajo/test-nhcb-over-rw2

Browse Source 
test(rw2): add nhcb testcases to remote write 2.0
This commit is contained in:
George Krajcsovits 2025-04-09 12:39:17 +02:00 committed by GitHub
commit b4526c0735
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 79 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
prompb/io/prometheus/write/v2/codec.go
View File

@ -196,6 +196,9 @@ func FromFloatHistogram(timestamp int64, fh *histogram.FloatHistogram) Histogram
} }
func spansToSpansProto(s []histogram.Span) []BucketSpan { func spansToSpansProto(s []histogram.Span) []BucketSpan {
if len(s) == 0 {
return nil
}
spans := make([]BucketSpan, len(s)) spans := make([]BucketSpan, len(s))
for i := 0; i < len(s); i++ { for i := 0; i < len(s); i++ {
spans[i] = BucketSpan{Offset: s[i].Offset, Length: s[i].Length} spans[i] = BucketSpan{Offset: s[i].Offset, Length: s[i].Length}

78
storage/remote/codec_test.go
View File

@ -43,12 +43,12 @@ var (
Schema: 2, Schema: 2,
ZeroThreshold: 1e-128, ZeroThreshold: 1e-128,
ZeroCount: 0, ZeroCount: 0,
Count: 0, Count: 3,
Sum: 20, Sum: 20,
PositiveSpans: []histogram.Span{{Offset: 0, Length: 1}}, PositiveSpans: []histogram.Span{{Offset: 0, Length: 1}},
PositiveBuckets: []int64{1}, PositiveBuckets: []int64{1},
NegativeSpans: []histogram.Span{{Offset: 0, Length: 1}}, NegativeSpans: []histogram.Span{{Offset: 0, Length: 1}},
NegativeBuckets: []int64{-1}, NegativeBuckets: []int64{2},
} }
writeRequestFixture = &prompb.WriteRequest{ writeRequestFixture = &prompb.WriteRequest{
@ -90,6 +90,15 @@ var (
Help: "Test counter for test purposes", Help: "Test counter for test purposes",
} }
testHistogramCustomBuckets = histogram.Histogram{
Schema: histogram.CustomBucketsSchema,
Count: 16,
Sum: 20,
PositiveSpans: []histogram.Span{{Offset: 1, Length: 2}},
PositiveBuckets: []int64{10, -4}, // Means 10 observations for upper bound 1.0 and 6 for upper bound +Inf.
CustomValues: []float64{0.1, 1.0}, // +Inf is implied.
}
// writeV2RequestFixture represents the same request as writeRequestFixture, // writeV2RequestFixture represents the same request as writeRequestFixture,
// but using the v2 representation, plus includes writeV2RequestSeries1Metadata and writeV2RequestSeries2Metadata. // but using the v2 representation, plus includes writeV2RequestSeries1Metadata and writeV2RequestSeries2Metadata.
// NOTE: Use TestWriteV2RequestFixture and copy the diff to regenerate if needed. // NOTE: Use TestWriteV2RequestFixture and copy the diff to regenerate if needed.
@ -104,9 +113,14 @@ var (
HelpRef: 15, // Symbolized writeV2RequestSeries1Metadata.Help. HelpRef: 15, // Symbolized writeV2RequestSeries1Metadata.Help.
UnitRef: 16, // Symbolized writeV2RequestSeries1Metadata.Unit. UnitRef: 16, // Symbolized writeV2RequestSeries1Metadata.Unit.
}, },
Samples: []writev2.Sample{{Value: 1, Timestamp: 10}}, Samples: []writev2.Sample{{Value: 1, Timestamp: 10}},
Exemplars: []writev2.Exemplar{{LabelsRefs: []uint32{11, 12}, Value: 1, Timestamp: 10}}, Exemplars: []writev2.Exemplar{{LabelsRefs: []uint32{11, 12}, Value: 1, Timestamp: 10}},
Histograms: []writev2.Histogram{writev2.FromIntHistogram(10, &testHistogram), writev2.FromFloatHistogram(20, testHistogram.ToFloat(nil))}, Histograms: []writev2.Histogram{
writev2.FromIntHistogram(10, &testHistogram),
writev2.FromFloatHistogram(20, testHistogram.ToFloat(nil)),
writev2.FromIntHistogram(30, &testHistogramCustomBuckets),
writev2.FromFloatHistogram(40, testHistogramCustomBuckets.ToFloat(nil)),
},
CreatedTimestamp: 1, // CT needs to be lower than the sample's timestamp. CreatedTimestamp: 1, // CT needs to be lower than the sample's timestamp.
}, },
{ {
@ -117,14 +131,40 @@ var (
HelpRef: 17, // Symbolized writeV2RequestSeries2Metadata.Help. HelpRef: 17, // Symbolized writeV2RequestSeries2Metadata.Help.
// No unit. // No unit.
}, },
Samples: []writev2.Sample{{Value: 2, Timestamp: 20}}, Samples: []writev2.Sample{{Value: 2, Timestamp: 20}},
Exemplars: []writev2.Exemplar{{LabelsRefs: []uint32{13, 14}, Value: 2, Timestamp: 20}}, Exemplars: []writev2.Exemplar{{LabelsRefs: []uint32{13, 14}, Value: 2, Timestamp: 20}},
Histograms: []writev2.Histogram{writev2.FromIntHistogram(30, &testHistogram), writev2.FromFloatHistogram(40, testHistogram.ToFloat(nil))}, Histograms: []writev2.Histogram{
writev2.FromIntHistogram(50, &testHistogram),
writev2.FromFloatHistogram(60, testHistogram.ToFloat(nil)),
writev2.FromIntHistogram(70, &testHistogramCustomBuckets),
writev2.FromFloatHistogram(80, testHistogramCustomBuckets.ToFloat(nil)),
},
}, },
}, },
} }
) )
func TestHistogramFixtureValid(t *testing.T) {
for _, ts := range writeRequestFixture.Timeseries {
for _, h := range ts.Histograms {
if h.IsFloatHistogram() {
require.NoError(t, h.ToFloatHistogram().Validate())
} else {
require.NoError(t, h.ToIntHistogram().Validate())
}
}
}
for _, ts := range writeV2RequestFixture.Timeseries {
for _, h := range ts.Histograms {
if h.IsFloatHistogram() {
require.NoError(t, h.ToFloatHistogram().Validate())
} else {
require.NoError(t, h.ToIntHistogram().Validate())
}
}
}
}
func TestWriteV2RequestFixture(t *testing.T) { func TestWriteV2RequestFixture(t *testing.T) {
// Generate dynamically writeV2RequestFixture, reusing v1 fixture elements. // Generate dynamically writeV2RequestFixture, reusing v1 fixture elements.
st := writev2.NewSymbolTable() st := writev2.NewSymbolTable()
@ -141,9 +181,14 @@ func TestWriteV2RequestFixture(t *testing.T) {
HelpRef: st.Symbolize(writeV2RequestSeries1Metadata.Help), HelpRef: st.Symbolize(writeV2RequestSeries1Metadata.Help),
UnitRef: st.Symbolize(writeV2RequestSeries1Metadata.Unit), UnitRef: st.Symbolize(writeV2RequestSeries1Metadata.Unit),
}, },
Samples: []writev2.Sample{{Value: 1, Timestamp: 10}}, Samples: []writev2.Sample{{Value: 1, Timestamp: 10}},
Exemplars: []writev2.Exemplar{{LabelsRefs: exemplar1LabelRefs, Value: 1, Timestamp: 10}}, Exemplars: []writev2.Exemplar{{LabelsRefs: exemplar1LabelRefs, Value: 1, Timestamp: 10}},
Histograms: []writev2.Histogram{writev2.FromIntHistogram(10, &testHistogram), writev2.FromFloatHistogram(20, testHistogram.ToFloat(nil))}, Histograms: []writev2.Histogram{
writev2.FromIntHistogram(10, &testHistogram),
writev2.FromFloatHistogram(20, testHistogram.ToFloat(nil)),
writev2.FromIntHistogram(30, &testHistogramCustomBuckets),
writev2.FromFloatHistogram(40, testHistogramCustomBuckets.ToFloat(nil)),
},
CreatedTimestamp: 1, CreatedTimestamp: 1,
}, },
{ {
@ -153,9 +198,14 @@ func TestWriteV2RequestFixture(t *testing.T) {
HelpRef: st.Symbolize(writeV2RequestSeries2Metadata.Help), HelpRef: st.Symbolize(writeV2RequestSeries2Metadata.Help),
// No unit. // No unit.
}, },
Samples: []writev2.Sample{{Value: 2, Timestamp: 20}}, Samples: []writev2.Sample{{Value: 2, Timestamp: 20}},
Exemplars: []writev2.Exemplar{{LabelsRefs: exemplar2LabelRefs, Value: 2, Timestamp: 20}}, Exemplars: []writev2.Exemplar{{LabelsRefs: exemplar2LabelRefs, Value: 2, Timestamp: 20}},
Histograms: []writev2.Histogram{writev2.FromIntHistogram(30, &testHistogram), writev2.FromFloatHistogram(40, testHistogram.ToFloat(nil))}, Histograms: []writev2.Histogram{
writev2.FromIntHistogram(50, &testHistogram),
writev2.FromFloatHistogram(60, testHistogram.ToFloat(nil)),
writev2.FromIntHistogram(70, &testHistogramCustomBuckets),
writev2.FromFloatHistogram(80, testHistogramCustomBuckets.ToFloat(nil)),
},
}, },
}, },
Symbols: st.Symbols(), Symbols: st.Symbols(),

16
storage/remote/write_handler_test.go
View File

@ -391,7 +391,7 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) {
desc: "Partial write; first series with one dup histogram sample", desc: "Partial write; first series with one dup histogram sample",
input: func() []writev2.TimeSeries { input: func() []writev2.TimeSeries {
f := proto.Clone(writeV2RequestFixture).(*writev2.Request) f := proto.Clone(writeV2RequestFixture).(*writev2.Request)
f.Timeseries[0].Histograms = append(f.Timeseries[0].Histograms, f.Timeseries[0].Histograms[1]) f.Timeseries[0].Histograms = append(f.Timeseries[0].Histograms, f.Timeseries[0].Histograms[len(f.Timeseries[0].Histograms)-1])
return f.Timeseries return f.Timeseries
}(), }(),
expectedCode: http.StatusBadRequest, expectedCode: http.StatusBadRequest,
@ -483,7 +483,7 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) {
// Double check mandatory 2.0 stats. // Double check mandatory 2.0 stats.
// writeV2RequestFixture has 2 series with 1 sample, 2 histograms, 1 exemplar each. // writeV2RequestFixture has 2 series with 1 sample, 2 histograms, 1 exemplar each.
expectHeaderValue(t, 2, resp.Header.Get(rw20WrittenSamplesHeader)) expectHeaderValue(t, 2, resp.Header.Get(rw20WrittenSamplesHeader))
expectHeaderValue(t, 4, resp.Header.Get(rw20WrittenHistogramsHeader)) expectHeaderValue(t, 8, resp.Header.Get(rw20WrittenHistogramsHeader))
if tc.appendExemplarErr != nil { if tc.appendExemplarErr != nil {
expectHeaderValue(t, 0, resp.Header.Get(rw20WrittenExemplarsHeader)) expectHeaderValue(t, 0, resp.Header.Get(rw20WrittenExemplarsHeader))
} else { } else {
@ -496,6 +496,8 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) {
i, j, k, m int i, j, k, m int
) )
for _, ts := range writeV2RequestFixture.Timeseries { for _, ts := range writeV2RequestFixture.Timeseries {
zeroHistogramIngested := false
zeroFloatHistogramIngested := false
ls := ts.ToLabels(&b, writeV2RequestFixture.Symbols) ls := ts.ToLabels(&b, writeV2RequestFixture.Symbols)
for _, s := range ts.Samples { for _, s := range ts.Samples {
@ -509,21 +511,27 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) {
for _, hp := range ts.Histograms { for _, hp := range ts.Histograms {
if hp.IsFloatHistogram() { if hp.IsFloatHistogram() {
fh := hp.ToFloatHistogram() fh := hp.ToFloatHistogram()
if ts.CreatedTimestamp != 0 && tc.ingestCTZeroSample { if !zeroFloatHistogramIngested && ts.CreatedTimestamp != 0 && tc.ingestCTZeroSample {
requireEqual(t, mockHistogram{ls, ts.CreatedTimestamp, nil, &histogram.FloatHistogram{}}, appendable.histograms[k]) requireEqual(t, mockHistogram{ls, ts.CreatedTimestamp, nil, &histogram.FloatHistogram{}}, appendable.histograms[k])
k++ k++
zeroFloatHistogramIngested = true
} }
requireEqual(t, mockHistogram{ls, hp.Timestamp, nil, fh}, appendable.histograms[k]) requireEqual(t, mockHistogram{ls, hp.Timestamp, nil, fh}, appendable.histograms[k])
} else { } else {
h := hp.ToIntHistogram() h := hp.ToIntHistogram()
if ts.CreatedTimestamp != 0 && tc.ingestCTZeroSample { if !zeroHistogramIngested && ts.CreatedTimestamp != 0 && tc.ingestCTZeroSample {
requireEqual(t, mockHistogram{ls, ts.CreatedTimestamp, &histogram.Histogram{}, nil}, appendable.histograms[k]) requireEqual(t, mockHistogram{ls, ts.CreatedTimestamp, &histogram.Histogram{}, nil}, appendable.histograms[k])
k++ k++
zeroHistogramIngested = true
} }
requireEqual(t, mockHistogram{ls, hp.Timestamp, h, nil}, appendable.histograms[k]) requireEqual(t, mockHistogram{ls, hp.Timestamp, h, nil}, appendable.histograms[k])
} }
k++ k++
} }
if ts.CreatedTimestamp != 0 && tc.ingestCTZeroSample {
require.True(t, zeroHistogramIngested)
require.True(t, zeroFloatHistogramIngested)
}
if tc.appendExemplarErr == nil { if tc.appendExemplarErr == nil {
for _, e := range ts.Exemplars { for _, e := range ts.Exemplars {
exemplarLabels := e.ToExemplar(&b, writeV2RequestFixture.Symbols).Labels exemplarLabels := e.ToExemplar(&b, writeV2RequestFixture.Symbols).Labels