From acd9aa0afb59fb5542e8dba4c6692e2a8a0df102 Mon Sep 17 00:00:00 2001 From: George Krajcsovits Date: Mon, 8 Sep 2025 17:26:41 +0200 Subject: [PATCH] fix(textparse/protobuf): metric family name corrupted by NHCB parser (#17156) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(textparse): implement NHCB parsing in ProtoBuf parser directly The NHCB conversion does some validation, but we can only return error from Parser.Next() not Parser.Histogram(). So the conversion needs to happen in Next(). There are 2 cases: 1. "always_scrape_classic_histograms" is enabled, in which case we convert after returning the classic series. This is to be consistent with the PromParser text parser, which collects NHCB while spitting out classic series; then returns the NHCB. 2. "always_scrape_classic_histograms" is disabled. In which case we never return the classic series. Signed-off-by: György Krajcsovits * refactor(textparse): skip classic series instead of adding NHCB around Do not return the first classic series from the EntryType state, switch to EntrySeries. This means we need to start the histogram field state from -3 , not -2. In EntrySeries, skip classic series if needed. Signed-off-by: György Krajcsovits * reuse nhcb converter Signed-off-by: György Krajcsovits * test(textparse/nhcb): test corrupting metric family name NHCB parse doesn't always copy the metric name from the underlying parser. When called via HELP, UNIT, the string is directly referenced which means that the read-ahead of NHCB can corrupt it. Signed-off-by: György Krajcsovits --- model/labels/labels_common.go | 13 +- model/textparse/benchmark_test.go | 4 +- model/textparse/interface.go | 2 +- model/textparse/nhcbparse.go | 12 +- model/textparse/nhcbparse_test.go | 137 ++- model/textparse/protobufparse.go | 134 ++- model/textparse/protobufparse_test.go | 1177 ++++++++++++++++++++++++- web/federate_test.go | 2 +- 8 files changed, 1391 insertions(+), 90 deletions(-) diff --git a/model/labels/labels_common.go b/model/labels/labels_common.go index 8345c12d16..e27da94a47 100644 --- a/model/labels/labels_common.go +++ b/model/labels/labels_common.go @@ -43,6 +43,15 @@ type Label struct { } func (ls Labels) String() string { + return ls.stringImpl(true) +} + +// StringNoSpace is like String but does not add a space after commas. +func (ls Labels) StringNoSpace() string { + return ls.stringImpl(false) +} + +func (ls Labels) stringImpl(addSpace bool) string { var bytea [1024]byte // On stack to avoid memory allocation while building the output. b := bytes.NewBuffer(bytea[:0]) @@ -51,7 +60,9 @@ func (ls Labels) String() string { ls.Range(func(l Label) { if i > 0 { b.WriteByte(',') - b.WriteByte(' ') + if addSpace { + b.WriteByte(' ') + } } if !model.LegacyValidation.IsValidLabelName(l.Name) { b.Write(strconv.AppendQuote(b.AvailableBuffer(), l.Name)) diff --git a/model/textparse/benchmark_test.go b/model/textparse/benchmark_test.go index cd3f332a6d..59ca349e31 100644 --- a/model/textparse/benchmark_test.go +++ b/model/textparse/benchmark_test.go @@ -149,7 +149,7 @@ func benchParse(b *testing.B, data []byte, parser string) { } case "promproto": newParserFn = func(b []byte, st *labels.SymbolTable) Parser { - return NewProtobufParser(b, true, false, st) + return NewProtobufParser(b, true, false, false, st) } case "omtext": newParserFn = func(b []byte, st *labels.SymbolTable) Parser { @@ -276,7 +276,7 @@ func BenchmarkCreatedTimestampPromProto(b *testing.B) { data := createTestProtoBuf(b).Bytes() st := labels.NewSymbolTable() - p := NewProtobufParser(data, true, false, st) + p := NewProtobufParser(data, true, false, false, st) found := false Inner: diff --git a/model/textparse/interface.go b/model/textparse/interface.go index 2bc2859ee7..c4b0aad5e8 100644 --- a/model/textparse/interface.go +++ b/model/textparse/interface.go @@ -142,7 +142,7 @@ func New(b []byte, contentType, fallbackType string, parseClassicHistograms, con o.enableTypeAndUnitLabels = enableTypeAndUnitLabels }) case "application/vnd.google.protobuf": - baseParser = NewProtobufParser(b, parseClassicHistograms, enableTypeAndUnitLabels, st) + return NewProtobufParser(b, parseClassicHistograms, convertClassicHistogramsToNHCB, enableTypeAndUnitLabels, st), err case "text/plain": baseParser = NewPromParser(b, st, enableTypeAndUnitLabels) default: diff --git a/model/textparse/nhcbparse.go b/model/textparse/nhcbparse.go index e7cfcc028e..d820a0f8b1 100644 --- a/model/textparse/nhcbparse.go +++ b/model/textparse/nhcbparse.go @@ -18,7 +18,6 @@ import ( "io" "math" "strconv" - "strings" "github.com/prometheus/common/model" @@ -373,7 +372,16 @@ func (p *NHCBParser) processNHCB() bool { p.hNHCB = nil p.fhNHCB = fh } - p.metricStringNHCB = p.tempLsetNHCB.Get(labels.MetricName) + strings.ReplaceAll(p.tempLsetNHCB.DropMetricName().String(), ", ", ",") + + lblsWithMetricName := p.tempLsetNHCB.DropMetricName() + // Ensure we return `metric` instead of `metric{}` for name only + // series, for consistency with wrapped parsers. + if lblsWithMetricName.IsEmpty() { + p.metricStringNHCB = p.tempLsetNHCB.Get(labels.MetricName) + } else { + p.metricStringNHCB = p.tempLsetNHCB.Get(labels.MetricName) + lblsWithMetricName.StringNoSpace() + } + p.bytesNHCB = []byte(p.metricStringNHCB) p.lsetNHCB = p.tempLsetNHCB p.swapExemplars() diff --git a/model/textparse/nhcbparse_test.go b/model/textparse/nhcbparse_test.go index d5d38e4edf..f3f5b9c444 100644 --- a/model/textparse/nhcbparse_test.go +++ b/model/textparse/nhcbparse_test.go @@ -15,18 +15,15 @@ package textparse import ( "bytes" - "encoding/binary" "strconv" "testing" - "github.com/gogo/protobuf/proto" "github.com/prometheus/common/model" "github.com/stretchr/testify/require" "github.com/prometheus/prometheus/model/exemplar" "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" - dto "github.com/prometheus/prometheus/prompb/io/prometheus/client" ) func TestNHCBParserOnOMParser(t *testing.T) { @@ -182,7 +179,7 @@ foobar{quantile="0.99"} 150.1` m: "hh", typ: model.MetricTypeHistogram, }, { - m: `hh{}`, + m: `hh`, shs: &histogram.Histogram{ Schema: histogram.CustomBucketsSchema, Count: 1, @@ -203,7 +200,7 @@ foobar{quantile="0.99"} 150.1` m: "hhh", typ: model.MetricTypeHistogram, }, { - m: `hhh{}`, + m: `hhh`, shs: &histogram.Histogram{ Schema: histogram.CustomBucketsSchema, Count: 1, @@ -333,7 +330,7 @@ foobar{quantile="0.99"} 150.1` m: "baz", typ: model.MetricTypeHistogram, }, { - m: `baz{}`, + m: `baz`, shs: &histogram.Histogram{ Schema: histogram.CustomBucketsSchema, Count: 17, @@ -361,7 +358,7 @@ foobar{quantile="0.99"} 150.1` m: "something", typ: model.MetricTypeHistogram, }, { - m: `something{}`, + m: `something`, shs: &histogram.Histogram{ Schema: histogram.CustomBucketsSchema, Count: 18, @@ -479,7 +476,7 @@ something_bucket{a="b",le="+Inf"} 9 # {id="something-test"} 2e100 123.000 m: "something", typ: model.MetricTypeHistogram, }, { - m: `something{}`, + m: `something`, shs: &histogram.Histogram{ Schema: histogram.CustomBucketsSchema, Count: 18, @@ -737,7 +734,7 @@ func TestNHCBParser_NoNHCBWhenExponential(t *testing.T) { // Always expect NHCB series after classic. nhcbSeries := []parsedEntry{ { - m: metric + "{}", + m: metric, shs: &histogram.Histogram{ Schema: histogram.CustomBucketsSchema, Count: 175, @@ -893,24 +890,7 @@ metric: < > `} - varintBuf := make([]byte, binary.MaxVarintLen32) - buf := &bytes.Buffer{} - - for _, tmf := range testMetricFamilies { - pb := &dto.MetricFamily{} - // From text to proto message. - require.NoError(t, proto.UnmarshalText(tmf, pb)) - // From proto message to binary protobuf. - protoBuf, err := proto.Marshal(pb) - require.NoError(t, err) - - // Write first length, then binary protobuf. - varintLength := binary.PutUvarint(varintBuf, uint64(len(protoBuf))) - buf.Write(varintBuf[:varintLength]) - buf.Write(protoBuf) - } - - return buf + return metricFamiliesToProtobuf(t, testMetricFamilies) } func createTestOpenMetricsHistogram() string { @@ -1054,22 +1034,7 @@ metric: < timestamp_ms: 1234568 >`} - varintBuf := make([]byte, binary.MaxVarintLen32) - buf := &bytes.Buffer{} - - for _, tmf := range testMetricFamilies { - pb := &dto.MetricFamily{} - // From text to proto message. - require.NoError(t, proto.UnmarshalText(tmf, pb)) - // From proto message to binary protobuf. - protoBuf, err := proto.Marshal(pb) - require.NoError(t, err) - - // Write first length, then binary protobuf. - varintLength := binary.PutUvarint(varintBuf, uint64(len(protoBuf))) - buf.Write(varintBuf[:varintLength]) - buf.Write(protoBuf) - } + buf := metricFamiliesToProtobuf(t, testMetricFamilies) exp := []parsedEntry{ { @@ -1107,7 +1072,7 @@ metric: < typ: model.MetricTypeHistogram, }, { - m: "test_histogram2{}", + m: "test_histogram2", shs: &histogram.Histogram{ Schema: histogram.CustomBucketsSchema, Count: 175, @@ -1128,3 +1093,87 @@ metric: < got := testParse(t, p) requireEntries(t, exp, got) } + +// TestNHCBNotCorruptMetricNameAfterRead is a regression test for https://github.com/prometheus/prometheus/issues/17075. +func TestNHCBNotCorruptMetricNameAfterRead(t *testing.T) { + inputOM := `# HELP test_histogram_seconds Just a test histogram +# TYPE test_histogram_seconds histogram +test_histogram_seconds_count 10 +test_histogram_seconds_sum 100 +test_histogram_seconds_bucket{le="10"} 10 +test_histogram_seconds_bucket{le="+Inf"} 10 +# HELP different_metric Just a different metric +# TYPE different_metric histogram +different_metric_count 5 +different_metric_sum 50 +different_metric_bucket{le="10"} 5 +different_metric_bucket{le="+Inf"} 5 +# EOF` + + testMetricFamilies := []string{`name: "test_histogram_seconds" +help: "Just a test histogram" +type: HISTOGRAM +metric: < + histogram: < + sample_count: 10 + sample_sum: 100 + bucket: < + cumulative_count: 10 + upper_bound: 10 + > + > +>`, `name: "different_metric" +help: "Just a different metric" +type: HISTOGRAM +metric: < + histogram: < + sample_count: 5 + sample_sum: 50 + bucket: < + cumulative_count: 5 + upper_bound: 10 + > + > +>`} + + buf := metricFamiliesToProtobuf(t, testMetricFamilies) + + testCases := []struct { + input []byte + typ string + }{ + {input: buf.Bytes(), typ: "application/vnd.google.protobuf"}, + {input: []byte(inputOM), typ: "text/plain"}, + {input: []byte(inputOM), typ: "application/openmetrics-text"}, + } + + for _, tc := range testCases { + t.Run(tc.typ, func(t *testing.T) { + p, err := New(tc.input, tc.typ, "", false, true, false, false, labels.NewSymbolTable()) + require.NoError(t, err) + require.NotNil(t, p) + + getNext := func() Entry { + e, err := p.Next() + require.NoError(t, err) + return e + } + + require.Equal(t, EntryHelp, getNext()) + lastMFName, lastHelp := p.Help() + require.Equal(t, "test_histogram_seconds", string(lastMFName)) + require.Equal(t, "Just a test histogram", string(lastHelp)) + + require.Equal(t, EntryType, getNext()) + var lastType model.MetricType + lastMFName, lastType = p.Type() + require.Equal(t, "test_histogram_seconds", string(lastMFName)) + require.Equal(t, model.MetricTypeHistogram, lastType) + + require.Equal(t, EntryHistogram, getNext()) + _, _, h, _ := p.Histogram() + require.NotNil(t, h) + require.Equal(t, "test_histogram_seconds", string(lastMFName)) + }) + } +} diff --git a/model/textparse/protobufparse.go b/model/textparse/protobufparse.go index e7ce710491..1b64a4d490 100644 --- a/model/textparse/protobufparse.go +++ b/model/textparse/protobufparse.go @@ -32,6 +32,7 @@ import ( "github.com/prometheus/prometheus/model/labels" dto "github.com/prometheus/prometheus/prompb/io/prometheus/client" "github.com/prometheus/prometheus/schema" + "github.com/prometheus/prometheus/util/convertnhcb" ) // floatFormatBufPool is exclusively used in formatOpenMetricsFloat. @@ -78,20 +79,31 @@ type ProtobufParser struct { // Whether to also parse a classic histogram that is also present as a // native histogram. - parseClassicHistograms bool + parseClassicHistograms bool + // Whether to add type and unit labels. enableTypeAndUnitLabels bool + + // Whether to convert classic histograms to native histograms with custom buckets. + convertClassicHistogramsToNHCB bool + // Reusable classic to NHCB converter. + tmpNHCB convertnhcb.TempHistogram + // We need to preload NHCB since we cannot do error handling in Histogram(). + nhcbH *histogram.Histogram + nhcbFH *histogram.FloatHistogram } // NewProtobufParser returns a parser for the payload in the byte slice. -func NewProtobufParser(b []byte, parseClassicHistograms, enableTypeAndUnitLabels bool, st *labels.SymbolTable) Parser { +func NewProtobufParser(b []byte, parseClassicHistograms, convertClassicHistogramsToNHCB, enableTypeAndUnitLabels bool, st *labels.SymbolTable) Parser { return &ProtobufParser{ dec: dto.NewMetricStreamingDecoder(b), entryBytes: &bytes.Buffer{}, builder: labels.NewScratchBuilderWithSymbolTable(st, 16), // TODO(bwplotka): Try base builder. - state: EntryInvalid, - parseClassicHistograms: parseClassicHistograms, - enableTypeAndUnitLabels: enableTypeAndUnitLabels, + state: EntryInvalid, + parseClassicHistograms: parseClassicHistograms, + enableTypeAndUnitLabels: enableTypeAndUnitLabels, + convertClassicHistogramsToNHCB: convertClassicHistogramsToNHCB, + tmpNHCB: convertnhcb.NewTempHistogram(), } } @@ -182,6 +194,15 @@ func (p *ProtobufParser) Histogram() ([]byte, *int64, *histogram.Histogram, *his h = p.dec.GetHistogram() ) + if !isNativeHistogram(h) { + // This only happens if we have a classic histogram and + // we converted it to NHCB already in Next. + if *ts != 0 { + return p.entryBytes.Bytes(), ts, p.nhcbH, p.nhcbFH + } + return p.entryBytes.Bytes(), nil, p.nhcbH, p.nhcbFH + } + if p.parseClassicHistograms && len(h.GetBucket()) > 0 { p.redoClassic = true } @@ -406,6 +427,8 @@ func (p *ProtobufParser) CreatedTimestamp() int64 { // read. func (p *ProtobufParser) Next() (Entry, error) { p.exemplarReturned = false + p.nhcbH = nil + p.nhcbFH = nil switch p.state { // Invalid state occurs on: // * First Next() call. @@ -468,8 +491,12 @@ func (p *ProtobufParser) Next() (Entry, error) { p.state = EntryType case EntryType: t := p.dec.GetType() - if (t == dto.MetricType_HISTOGRAM || t == dto.MetricType_GAUGE_HISTOGRAM) && - isNativeHistogram(p.dec.GetHistogram()) { + if t == dto.MetricType_HISTOGRAM || t == dto.MetricType_GAUGE_HISTOGRAM { + if !isNativeHistogram(p.dec.GetHistogram()) { + p.state = EntrySeries + p.fieldPos = -3 // We have not returned anything, let p.Next() increment it to -2. + return p.Next() + } p.state = EntryHistogram } else { p.state = EntrySeries @@ -480,14 +507,18 @@ func (p *ProtobufParser) Next() (Entry, error) { case EntrySeries: // Potentially a second series in the metric family. t := p.dec.GetType() + decodeNext := true if t == dto.MetricType_SUMMARY || t == dto.MetricType_HISTOGRAM || t == dto.MetricType_GAUGE_HISTOGRAM { // Non-trivial series (complex metrics, with magic suffixes). + isClassicHistogram := (t == dto.MetricType_HISTOGRAM || t == dto.MetricType_GAUGE_HISTOGRAM) && !isNativeHistogram(p.dec.GetHistogram()) + skipSeries := p.convertClassicHistogramsToNHCB && isClassicHistogram && !p.parseClassicHistograms + // Did we iterate over all the classic representations fields? // NOTE: p.fieldsDone is updated on p.onSeriesOrHistogramUpdate. - if !p.fieldsDone { + if !p.fieldsDone && !skipSeries { // Still some fields to iterate over. p.fieldPos++ if err := p.onSeriesOrHistogramUpdate(); err != nil { @@ -504,25 +535,39 @@ func (p *ProtobufParser) Next() (Entry, error) { // If this is a metric family containing native // histograms, it means we are here thanks to redoClassic state. // Return to native histograms for the consistent flow. - if (t == dto.MetricType_HISTOGRAM || t == dto.MetricType_GAUGE_HISTOGRAM) && - isNativeHistogram(p.dec.GetHistogram()) { - p.state = EntryHistogram + // If this is a metric family containing classic histograms, + // it means we might need to do NHCB conversion. + if t == dto.MetricType_HISTOGRAM || t == dto.MetricType_GAUGE_HISTOGRAM { + if !isClassicHistogram { + p.state = EntryHistogram + } else if p.convertClassicHistogramsToNHCB { + // We still need to spit out the NHCB. + var err error + p.nhcbH, p.nhcbFH, err = p.convertToNHCB(t) + if err != nil { + return EntryInvalid, err + } + p.state = EntryHistogram + // We have an NHCB to emit, no need to decode the next series. + decodeNext = false + } } } // Is there another series? - if err := p.dec.NextMetric(); err != nil { - if errors.Is(err, io.EOF) { - p.state = EntryInvalid - return p.Next() + if decodeNext { + if err := p.dec.NextMetric(); err != nil { + if errors.Is(err, io.EOF) { + p.state = EntryInvalid + return p.Next() + } + return EntryInvalid, err } - return EntryInvalid, err } if err := p.onSeriesOrHistogramUpdate(); err != nil { return EntryInvalid, err } case EntryHistogram: - // Was Histogram() called and parseClassicHistograms is true? - if p.redoClassic { + switchToClassic := func() (Entry, error) { p.redoClassic = false p.fieldPos = -3 p.fieldsDone = false @@ -530,6 +575,11 @@ func (p *ProtobufParser) Next() (Entry, error) { return p.Next() // Switch to classic histogram. } + // Was Histogram() called and parseClassicHistograms is true? + if p.redoClassic { + return switchToClassic() + } + // Is there another series? if err := p.dec.NextMetric(); err != nil { if errors.Is(err, io.EOF) { @@ -538,6 +588,14 @@ func (p *ProtobufParser) Next() (Entry, error) { } return EntryInvalid, err } + + // If this is a metric family does not contain native + // histograms, it means we are here thanks to NHCB conversion. + // Return to classic histograms for the consistent flow. + if !isNativeHistogram(p.dec.GetHistogram()) { + return switchToClassic() + } + if err := p.onSeriesOrHistogramUpdate(); err != nil { return EntryInvalid, err } @@ -690,3 +748,43 @@ func isNativeHistogram(h *dto.Histogram) bool { h.GetZeroThreshold() > 0 || h.GetZeroCount() > 0 } + +func (p *ProtobufParser) convertToNHCB(t dto.MetricType) (*histogram.Histogram, *histogram.FloatHistogram, error) { + h := p.dec.GetHistogram() + p.tmpNHCB.Reset() + // TODO(krajorama): convertnhcb should support setting integer mode up + // front since we know it here. That would avoid the converter having + // to guess it based on counts. + v := h.GetSampleCountFloat() + if v == 0 { + v = float64(h.GetSampleCount()) + } + if err := p.tmpNHCB.SetCount(v); err != nil { + return nil, nil, err + } + + if err := p.tmpNHCB.SetSum(h.GetSampleSum()); err != nil { + return nil, nil, err + } + for _, b := range h.GetBucket() { + v := b.GetCumulativeCountFloat() + if v == 0 { + v = float64(b.GetCumulativeCount()) + } + if err := p.tmpNHCB.SetBucketCount(b.GetUpperBound(), v); err != nil { + return nil, nil, err + } + } + ch, cfh, err := p.tmpNHCB.Convert() + if err != nil { + return nil, nil, err + } + if t == dto.MetricType_GAUGE_HISTOGRAM { + if ch != nil { + ch.CounterResetHint = histogram.GaugeType + } else { + cfh.CounterResetHint = histogram.GaugeType + } + } + return ch, cfh, nil +} diff --git a/model/textparse/protobufparse_test.go b/model/textparse/protobufparse_test.go index 35a4238fdb..7a7eb6eec7 100644 --- a/model/textparse/protobufparse_test.go +++ b/model/textparse/protobufparse_test.go @@ -16,6 +16,7 @@ package textparse import ( "bytes" "encoding/binary" + "fmt" "testing" "github.com/gogo/protobuf/proto" @@ -28,6 +29,26 @@ import ( dto "github.com/prometheus/prometheus/prompb/io/prometheus/client" ) +func metricFamiliesToProtobuf(t testing.TB, testMetricFamilies []string) *bytes.Buffer { + varintBuf := make([]byte, binary.MaxVarintLen32) + buf := &bytes.Buffer{} + + for _, tmf := range testMetricFamilies { + pb := &dto.MetricFamily{} + // From text to proto message. + require.NoError(t, proto.UnmarshalText(tmf, pb)) + // From proto message to binary protobuf. + protoBuf, err := proto.Marshal(pb) + require.NoError(t, err) + + // Write first length, then binary protobuf. + varintLength := binary.PutUvarint(varintBuf, uint64(len(protoBuf))) + buf.Write(varintBuf[:varintLength]) + buf.Write(protoBuf) + } + return buf +} + func createTestProtoBuf(t testing.TB) *bytes.Buffer { t.Helper() @@ -803,24 +824,7 @@ metric: < `, } - varintBuf := make([]byte, binary.MaxVarintLen32) - buf := &bytes.Buffer{} - - for _, tmf := range testMetricFamilies { - pb := &dto.MetricFamily{} - // From text to proto message. - require.NoError(t, proto.UnmarshalText(tmf, pb)) - // From proto message to binary protobuf. - protoBuf, err := proto.Marshal(pb) - require.NoError(t, err) - - // Write first length, then binary protobuf. - varintLength := binary.PutUvarint(varintBuf, uint64(len(protoBuf))) - buf.Write(varintBuf[:varintLength]) - buf.Write(protoBuf) - } - - return buf + return metricFamiliesToProtobuf(t, testMetricFamilies) } func TestProtobufParse(t *testing.T) { @@ -833,7 +837,7 @@ func TestProtobufParse(t *testing.T) { }{ { name: "parseClassicHistograms=false/enableTypeAndUnitLabels=false", - parser: NewProtobufParser(inputBuf.Bytes(), false, false, labels.NewSymbolTable()), + parser: NewProtobufParser(inputBuf.Bytes(), false, false, false, labels.NewSymbolTable()), expected: []parsedEntry{ { m: "go_build_info", @@ -1468,7 +1472,7 @@ func TestProtobufParse(t *testing.T) { }, { name: "parseClassicHistograms=false/enableTypeAndUnitLabels=true", - parser: NewProtobufParser(inputBuf.Bytes(), false, true, labels.NewSymbolTable()), + parser: NewProtobufParser(inputBuf.Bytes(), false, false, true, labels.NewSymbolTable()), expected: []parsedEntry{ { m: "go_build_info", @@ -2140,7 +2144,7 @@ func TestProtobufParse(t *testing.T) { }, { name: "parseClassicHistograms=true/enableTypeAndUnitLabels=false", - parser: NewProtobufParser(inputBuf.Bytes(), true, false, labels.NewSymbolTable()), + parser: NewProtobufParser(inputBuf.Bytes(), true, false, false, labels.NewSymbolTable()), expected: []parsedEntry{ { m: "go_build_info", @@ -3214,3 +3218,1134 @@ func TestProtobufParse(t *testing.T) { }) } } + +// TestProtobufParseWithNHCB is only concerned with classic histograms. +func TestProtobufParseWithNHCB(t *testing.T) { + testMetricFamilies := []string{ + `name: "test_histogram1" +help: "Similar histogram as before but now without sparse buckets." +type: HISTOGRAM +metric: < + histogram: < + sample_count: 175 + sample_sum: 0.000828 + bucket: < + cumulative_count: 2 + upper_bound: -0.00048 + > + bucket: < + cumulative_count: 4 + upper_bound: -0.00038 + exemplar: < + label: < + name: "dummyID" + value: "59727" + > + value: -0.00038 + timestamp: < + seconds: 1625851153 + nanos: 146848499 + > + > + > + bucket: < + cumulative_count: 16 + upper_bound: 1 + exemplar: < + label: < + name: "dummyID" + value: "5617" + > + value: -0.000295 + > + > + schema: 0 + zero_threshold: 0 + > +> + +`, + `name: "test_histogram2_seconds" +help: "Similar histogram as before but now with integer buckets." +type: HISTOGRAM +unit: "seconds" +metric: < + histogram: < + sample_count: 6 + sample_sum: 50 + bucket: < + cumulative_count: 2 + upper_bound: -20 + > + bucket: < + cumulative_count: 4 + upper_bound: 20 + exemplar: < + label: < + name: "dummyID" + value: "59727" + > + value: 15 + timestamp: < + seconds: 1625851153 + nanos: 146848499 + > + > + > + bucket: < + cumulative_count: 6 + upper_bound: 30 + exemplar: < + label: < + name: "dummyID" + value: "5617" + > + value: 25 + timestamp: < + seconds: 1625851153 + nanos: 146848499 + > + > + > + schema: 0 + zero_threshold: 0 + > +> + +`, + `name: "test_histogram_family" +help: "Test histogram metric family with two very simple histograms." +type: HISTOGRAM +metric: < + label: < + name: "foo" + value: "bar" + > + histogram: < + sample_count: 5 + sample_sum: 12.1 + bucket: < + cumulative_count: 2 + upper_bound: 1.1 + > + bucket: < + cumulative_count: 3 + upper_bound: 2.2 + > + > +> +metric: < + label: < + name: "foo" + value: "baz" + > + histogram: < + sample_count: 6 + sample_sum: 13.1 + bucket: < + cumulative_count: 0 + upper_bound: 1.1 + > + bucket: < + cumulative_count: 5 + upper_bound: 2.2 + > + > +> + +`, + `name: "empty_histogram" +help: "A histogram without observations and with a zero threshold of zero but with a no-op span to identify it as a native histogram." +type: HISTOGRAM +metric: < + histogram: < + positive_span: < + offset: 0 + length: 0 + > + > +> + +`, + } + + buf := metricFamiliesToProtobuf(t, testMetricFamilies) + + data := buf.Bytes() + + testCases := []struct { + keepClassic bool + typeAndUnit bool + expected []parsedEntry + }{ + { + keepClassic: false, + typeAndUnit: false, + expected: []parsedEntry{ + { + m: "test_histogram1", + help: "Similar histogram as before but now without sparse buckets.", + }, + { + m: "test_histogram1", + typ: model.MetricTypeHistogram, + }, + { + m: "test_histogram1", + shs: &histogram.Histogram{ + CounterResetHint: histogram.UnknownCounterReset, + Count: 175, + Sum: 0.000828, + Schema: -53, + PositiveSpans: []histogram.Span{ + {Length: 4}, + }, + NegativeSpans: []histogram.Span{}, + PositiveBuckets: []int64{2, 0, 10, 147}, + CustomValues: []float64{-0.00048, -0.00038, 1}, + }, + lset: labels.FromStrings( + "__name__", "test_histogram1", + ), + es: []exemplar.Exemplar{ + {Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00038, HasTs: true, Ts: 1625851153146}, + // The second exemplar has no timestamp. + }, + }, + { + m: "test_histogram2_seconds", + help: "Similar histogram as before but now with integer buckets.", + }, + { + m: "test_histogram2_seconds", + unit: "seconds", + }, + { + m: "test_histogram2_seconds", + typ: model.MetricTypeHistogram, + }, + { + m: "test_histogram2_seconds", + shs: &histogram.Histogram{ + CounterResetHint: histogram.UnknownCounterReset, + Count: 6, + Sum: 50, + Schema: -53, + PositiveSpans: []histogram.Span{ + {Length: 3}, + }, + NegativeSpans: []histogram.Span{}, + PositiveBuckets: []int64{2, 0, 0}, + CustomValues: []float64{-20, 20, 30}, + }, + lset: labels.FromStrings( + "__name__", "test_histogram2_seconds", + ), + es: []exemplar.Exemplar{ + {Labels: labels.FromStrings("dummyID", "59727"), Value: 15, HasTs: true, Ts: 1625851153146}, + {Labels: labels.FromStrings("dummyID", "5617"), Value: 25, HasTs: true, Ts: 1625851153146}, + }, + }, + { + m: "test_histogram_family", + help: "Test histogram metric family with two very simple histograms.", + }, + { + m: "test_histogram_family", + typ: model.MetricTypeHistogram, + }, + { + m: "test_histogram_family\xfffoo\xffbar", + shs: &histogram.Histogram{ + CounterResetHint: histogram.UnknownCounterReset, + Count: 5, + Sum: 12.1, + Schema: -53, + PositiveSpans: []histogram.Span{ + {Length: 3}, + }, + NegativeSpans: []histogram.Span{}, + PositiveBuckets: []int64{2, -1, 1}, + CustomValues: []float64{1.1, 2.2}, + }, + lset: labels.FromStrings( + "__name__", "test_histogram_family", + "foo", "bar", + ), + }, + { + m: "test_histogram_family\xfffoo\xffbaz", + shs: &histogram.Histogram{ + CounterResetHint: histogram.UnknownCounterReset, + Count: 6, + Sum: 13.1, + Schema: -53, + PositiveSpans: []histogram.Span{ + {Offset: 1, Length: 2}, + }, + NegativeSpans: []histogram.Span{}, + PositiveBuckets: []int64{5, -4}, + CustomValues: []float64{1.1, 2.2}, + }, + lset: labels.FromStrings( + "__name__", "test_histogram_family", + "foo", "baz", + ), + }, + { + m: "empty_histogram", + help: "A histogram without observations and with a zero threshold of zero but with a no-op span to identify it as a native histogram.", + }, + { + m: "empty_histogram", + typ: model.MetricTypeHistogram, + }, + { + m: "empty_histogram", + shs: &histogram.Histogram{ + CounterResetHint: histogram.UnknownCounterReset, + PositiveSpans: []histogram.Span{}, + NegativeSpans: []histogram.Span{}, + }, + lset: labels.FromStrings( + "__name__", "empty_histogram", + ), + }, + }, + }, + { + keepClassic: true, + typeAndUnit: false, + expected: []parsedEntry{ + { + m: "test_histogram1", + help: "Similar histogram as before but now without sparse buckets.", + }, + { + m: "test_histogram1", + typ: model.MetricTypeHistogram, + }, + { + m: "test_histogram1_count", + v: 175, + lset: labels.FromStrings( + "__name__", "test_histogram1_count", + ), + }, + { + m: "test_histogram1_sum", + v: 0.000828, + lset: labels.FromStrings( + "__name__", "test_histogram1_sum", + ), + }, + { + m: "test_histogram1_bucket\xffle\xff-0.00048", + v: 2, + lset: labels.FromStrings( + "__name__", "test_histogram1_bucket", + "le", "-0.00048", + ), + }, + { + m: "test_histogram1_bucket\xffle\xff-0.00038", + v: 4, + lset: labels.FromStrings( + "__name__", "test_histogram1_bucket", + "le", "-0.00038", + ), + es: []exemplar.Exemplar{ + {Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00038, HasTs: true, Ts: 1625851153146}, + }, + }, + { + m: "test_histogram1_bucket\xffle\xff1.0", + v: 16, + lset: labels.FromStrings( + "__name__", "test_histogram1_bucket", + "le", "1.0", + ), + es: []exemplar.Exemplar{ + {Labels: labels.FromStrings("dummyID", "5617"), Value: -0.000295, HasTs: false}, + }, + }, + { + m: "test_histogram1_bucket\xffle\xff+Inf", + v: 175, + lset: labels.FromStrings( + "__name__", "test_histogram1_bucket", + "le", "+Inf", + ), + }, + { + m: "test_histogram1", + shs: &histogram.Histogram{ + CounterResetHint: histogram.UnknownCounterReset, + Count: 175, + Sum: 0.000828, + Schema: -53, + PositiveSpans: []histogram.Span{ + {Length: 4}, + }, + NegativeSpans: []histogram.Span{}, + PositiveBuckets: []int64{2, 0, 10, 147}, + CustomValues: []float64{-0.00048, -0.00038, 1}, + }, + lset: labels.FromStrings( + "__name__", "test_histogram1", + ), + es: []exemplar.Exemplar{ + {Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00038, HasTs: true, Ts: 1625851153146}, + // The second exemplar has no timestamp. + }, + }, + { + m: "test_histogram2_seconds", + help: "Similar histogram as before but now with integer buckets.", + }, + { + m: "test_histogram2_seconds", + unit: "seconds", + }, + { + m: "test_histogram2_seconds", + typ: model.MetricTypeHistogram, + }, + { + m: "test_histogram2_seconds_count", + v: 6, + lset: labels.FromStrings( + "__name__", "test_histogram2_seconds_count", + ), + }, + { + m: "test_histogram2_seconds_sum", + v: 50, + lset: labels.FromStrings( + "__name__", "test_histogram2_seconds_sum", + ), + }, + { + m: "test_histogram2_seconds_bucket\xffle\xff-20.0", + v: 2, + lset: labels.FromStrings( + "__name__", "test_histogram2_seconds_bucket", + "le", "-20.0", + ), + }, + { + m: "test_histogram2_seconds_bucket\xffle\xff20.0", + v: 4, + lset: labels.FromStrings( + "__name__", "test_histogram2_seconds_bucket", + "le", "20.0", + ), + es: []exemplar.Exemplar{ + {Labels: labels.FromStrings("dummyID", "59727"), Value: 15, HasTs: true, Ts: 1625851153146}, + }, + }, + { + m: "test_histogram2_seconds_bucket\xffle\xff30.0", + v: 6, + lset: labels.FromStrings( + "__name__", "test_histogram2_seconds_bucket", + "le", "30.0", + ), + es: []exemplar.Exemplar{ + {Labels: labels.FromStrings("dummyID", "5617"), Value: 25, HasTs: true, Ts: 1625851153146}, + }, + }, + { + m: "test_histogram2_seconds_bucket\xffle\xff+Inf", + v: 6, + lset: labels.FromStrings( + "__name__", "test_histogram2_seconds_bucket", + "le", "+Inf", + ), + }, + { + m: "test_histogram2_seconds", + shs: &histogram.Histogram{ + CounterResetHint: histogram.UnknownCounterReset, + Count: 6, + Sum: 50, + Schema: -53, + PositiveSpans: []histogram.Span{ + {Length: 3}, + }, + NegativeSpans: []histogram.Span{}, + PositiveBuckets: []int64{2, 0, 0}, + CustomValues: []float64{-20, 20, 30}, + }, + lset: labels.FromStrings( + "__name__", "test_histogram2_seconds", + ), + es: []exemplar.Exemplar{ + {Labels: labels.FromStrings("dummyID", "59727"), Value: 15, HasTs: true, Ts: 1625851153146}, + {Labels: labels.FromStrings("dummyID", "5617"), Value: 25, HasTs: true, Ts: 1625851153146}, + }, + }, + { + m: "test_histogram_family", + help: "Test histogram metric family with two very simple histograms.", + }, + { + m: "test_histogram_family", + typ: model.MetricTypeHistogram, + }, + { + m: "test_histogram_family_count\xfffoo\xffbar", + v: 5, + lset: labels.FromStrings( + "__name__", "test_histogram_family_count", + "foo", "bar", + ), + }, + { + m: "test_histogram_family_sum\xfffoo\xffbar", + v: 12.1, + lset: labels.FromStrings( + "__name__", "test_histogram_family_sum", + "foo", "bar", + ), + }, + { + m: "test_histogram_family_bucket\xfffoo\xffbar\xffle\xff1.1", + v: 2, + lset: labels.FromStrings( + "__name__", "test_histogram_family_bucket", + "foo", "bar", + "le", "1.1", + ), + }, + { + m: "test_histogram_family_bucket\xfffoo\xffbar\xffle\xff2.2", + v: 3, + lset: labels.FromStrings( + "__name__", "test_histogram_family_bucket", + "foo", "bar", + "le", "2.2", + ), + }, + { + m: "test_histogram_family_bucket\xfffoo\xffbar\xffle\xff+Inf", + v: 5, + lset: labels.FromStrings( + "__name__", "test_histogram_family_bucket", + "foo", "bar", + "le", "+Inf", + ), + }, + { + m: "test_histogram_family\xfffoo\xffbar", + shs: &histogram.Histogram{ + CounterResetHint: histogram.UnknownCounterReset, + Count: 5, + Sum: 12.1, + Schema: -53, + PositiveSpans: []histogram.Span{ + {Length: 3}, + }, + NegativeSpans: []histogram.Span{}, + PositiveBuckets: []int64{2, -1, 1}, + CustomValues: []float64{1.1, 2.2}, + }, + lset: labels.FromStrings( + "__name__", "test_histogram_family", + "foo", "bar", + ), + }, + { + m: "test_histogram_family_count\xfffoo\xffbaz", + v: 6, + lset: labels.FromStrings( + "__name__", "test_histogram_family_count", + "foo", "baz", + ), + }, + { + m: "test_histogram_family_sum\xfffoo\xffbaz", + v: 13.1, + lset: labels.FromStrings( + "__name__", "test_histogram_family_sum", + "foo", "baz", + ), + }, + { + m: "test_histogram_family_bucket\xfffoo\xffbaz\xffle\xff1.1", + v: 0, + lset: labels.FromStrings( + "__name__", "test_histogram_family_bucket", + "foo", "baz", + "le", "1.1", + ), + }, + { + m: "test_histogram_family_bucket\xfffoo\xffbaz\xffle\xff2.2", + v: 5, + lset: labels.FromStrings( + "__name__", "test_histogram_family_bucket", + "foo", "baz", + "le", "2.2", + ), + }, + { + m: "test_histogram_family_bucket\xfffoo\xffbaz\xffle\xff+Inf", + v: 6, + lset: labels.FromStrings( + "__name__", "test_histogram_family_bucket", + "foo", "baz", + "le", "+Inf", + ), + }, + { + m: "test_histogram_family\xfffoo\xffbaz", + shs: &histogram.Histogram{ + CounterResetHint: histogram.UnknownCounterReset, + Count: 6, + Sum: 13.1, + Schema: -53, + PositiveSpans: []histogram.Span{ + {Offset: 1, Length: 2}, + }, + NegativeSpans: []histogram.Span{}, + PositiveBuckets: []int64{5, -4}, + CustomValues: []float64{1.1, 2.2}, + }, + lset: labels.FromStrings( + "__name__", "test_histogram_family", + "foo", "baz", + ), + }, + { + m: "empty_histogram", + help: "A histogram without observations and with a zero threshold of zero but with a no-op span to identify it as a native histogram.", + }, + { + m: "empty_histogram", + typ: model.MetricTypeHistogram, + }, + { + m: "empty_histogram", + shs: &histogram.Histogram{ + CounterResetHint: histogram.UnknownCounterReset, + PositiveSpans: []histogram.Span{}, + NegativeSpans: []histogram.Span{}, + }, + lset: labels.FromStrings( + "__name__", "empty_histogram", + ), + }, + }, + }, + { + keepClassic: false, + typeAndUnit: true, + expected: []parsedEntry{ + { + m: "test_histogram1", + help: "Similar histogram as before but now without sparse buckets.", + }, + { + m: "test_histogram1", + typ: model.MetricTypeHistogram, + }, + { + m: "test_histogram1\xff__type__\xffhistogram", + shs: &histogram.Histogram{ + CounterResetHint: histogram.UnknownCounterReset, + Count: 175, + Sum: 0.000828, + Schema: -53, + PositiveSpans: []histogram.Span{ + {Length: 4}, + }, + NegativeSpans: []histogram.Span{}, + PositiveBuckets: []int64{2, 0, 10, 147}, + CustomValues: []float64{-0.00048, -0.00038, 1}, + }, + lset: labels.FromStrings( + "__name__", "test_histogram1", + "__type__", "histogram", + ), + es: []exemplar.Exemplar{ + {Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00038, HasTs: true, Ts: 1625851153146}, + // The second exemplar has no timestamp. + }, + }, + { + m: "test_histogram2_seconds", + help: "Similar histogram as before but now with integer buckets.", + }, + { + m: "test_histogram2_seconds", + unit: "seconds", + }, + { + m: "test_histogram2_seconds", + typ: model.MetricTypeHistogram, + }, + { + m: "test_histogram2_seconds\xff__type__\xffhistogram\xff__unit__\xffseconds", + shs: &histogram.Histogram{ + CounterResetHint: histogram.UnknownCounterReset, + Count: 6, + Sum: 50, + Schema: -53, + PositiveSpans: []histogram.Span{ + {Length: 3}, + }, + NegativeSpans: []histogram.Span{}, + PositiveBuckets: []int64{2, 0, 0}, + CustomValues: []float64{-20, 20, 30}, + }, + lset: labels.FromStrings( + "__name__", "test_histogram2_seconds", + "__type__", "histogram", + "__unit__", "seconds", + ), + es: []exemplar.Exemplar{ + {Labels: labels.FromStrings("dummyID", "59727"), Value: 15, HasTs: true, Ts: 1625851153146}, + {Labels: labels.FromStrings("dummyID", "5617"), Value: 25, HasTs: true, Ts: 1625851153146}, + }, + }, + { + m: "test_histogram_family", + help: "Test histogram metric family with two very simple histograms.", + }, + { + m: "test_histogram_family", + typ: model.MetricTypeHistogram, + }, + { + m: "test_histogram_family\xff__type__\xffhistogram\xfffoo\xffbar", + shs: &histogram.Histogram{ + CounterResetHint: histogram.UnknownCounterReset, + Count: 5, + Sum: 12.1, + Schema: -53, + PositiveSpans: []histogram.Span{ + {Length: 3}, + }, + NegativeSpans: []histogram.Span{}, + PositiveBuckets: []int64{2, -1, 1}, + CustomValues: []float64{1.1, 2.2}, + }, + lset: labels.FromStrings( + "__name__", "test_histogram_family", + "__type__", "histogram", + "foo", "bar", + ), + }, + { + m: "test_histogram_family\xff__type__\xffhistogram\xfffoo\xffbaz", + shs: &histogram.Histogram{ + CounterResetHint: histogram.UnknownCounterReset, + Count: 6, + Sum: 13.1, + Schema: -53, + PositiveSpans: []histogram.Span{ + {Offset: 1, Length: 2}, + }, + NegativeSpans: []histogram.Span{}, + PositiveBuckets: []int64{5, -4}, + CustomValues: []float64{1.1, 2.2}, + }, + lset: labels.FromStrings( + "__name__", "test_histogram_family", + "__type__", "histogram", + "foo", "baz", + ), + }, + { + m: "empty_histogram", + help: "A histogram without observations and with a zero threshold of zero but with a no-op span to identify it as a native histogram.", + }, + { + m: "empty_histogram", + typ: model.MetricTypeHistogram, + }, + { + m: "empty_histogram\xff__type__\xffhistogram", + shs: &histogram.Histogram{ + CounterResetHint: histogram.UnknownCounterReset, + PositiveSpans: []histogram.Span{}, + NegativeSpans: []histogram.Span{}, + }, + lset: labels.FromStrings( + "__name__", "empty_histogram", + "__type__", "histogram", + ), + }, + }, + }, + { + keepClassic: true, + typeAndUnit: true, + expected: []parsedEntry{ + { + m: "test_histogram1", + help: "Similar histogram as before but now without sparse buckets.", + }, + { + m: "test_histogram1", + typ: model.MetricTypeHistogram, + }, + { + m: "test_histogram1_count\xff__type__\xffhistogram", + v: 175, + lset: labels.FromStrings( + "__name__", "test_histogram1_count", + "__type__", "histogram", + ), + }, + { + m: "test_histogram1_sum\xff__type__\xffhistogram", + v: 0.000828, + lset: labels.FromStrings( + "__name__", "test_histogram1_sum", + "__type__", "histogram", + ), + }, + { + m: "test_histogram1_bucket\xff__type__\xffhistogram\xffle\xff-0.00048", + v: 2, + lset: labels.FromStrings( + "__name__", "test_histogram1_bucket", + "__type__", "histogram", + "le", "-0.00048", + ), + }, + { + m: "test_histogram1_bucket\xff__type__\xffhistogram\xffle\xff-0.00038", + v: 4, + lset: labels.FromStrings( + "__name__", "test_histogram1_bucket", + "__type__", "histogram", + "le", "-0.00038", + ), + es: []exemplar.Exemplar{ + {Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00038, HasTs: true, Ts: 1625851153146}, + }, + }, + { + m: "test_histogram1_bucket\xff__type__\xffhistogram\xffle\xff1.0", + v: 16, + lset: labels.FromStrings( + "__name__", "test_histogram1_bucket", + "__type__", "histogram", + "le", "1.0", + ), + es: []exemplar.Exemplar{ + {Labels: labels.FromStrings("dummyID", "5617"), Value: -0.000295, HasTs: false}, + }, + }, + { + m: "test_histogram1_bucket\xff__type__\xffhistogram\xffle\xff+Inf", + v: 175, + lset: labels.FromStrings( + "__name__", "test_histogram1_bucket", + "__type__", "histogram", + "le", "+Inf", + ), + }, + { + m: "test_histogram1\xff__type__\xffhistogram", + shs: &histogram.Histogram{ + CounterResetHint: histogram.UnknownCounterReset, + Count: 175, + Sum: 0.000828, + Schema: -53, + PositiveSpans: []histogram.Span{ + {Length: 4}, + }, + NegativeSpans: []histogram.Span{}, + PositiveBuckets: []int64{2, 0, 10, 147}, + CustomValues: []float64{-0.00048, -0.00038, 1}, + }, + lset: labels.FromStrings( + "__name__", "test_histogram1", + "__type__", "histogram", + ), + es: []exemplar.Exemplar{ + {Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00038, HasTs: true, Ts: 1625851153146}, + // The second exemplar has no timestamp. + }, + }, + { + m: "test_histogram2_seconds", + help: "Similar histogram as before but now with integer buckets.", + }, + { + m: "test_histogram2_seconds", + unit: "seconds", + }, + { + m: "test_histogram2_seconds", + typ: model.MetricTypeHistogram, + }, + { + m: "test_histogram2_seconds_count\xff__type__\xffhistogram\xff__unit__\xffseconds", + v: 6, + lset: labels.FromStrings( + "__name__", "test_histogram2_seconds_count", + "__type__", "histogram", + "__unit__", "seconds", + ), + }, + { + m: "test_histogram2_seconds_sum\xff__type__\xffhistogram\xff__unit__\xffseconds", + v: 50, + lset: labels.FromStrings( + "__name__", "test_histogram2_seconds_sum", + "__type__", "histogram", + "__unit__", "seconds", + ), + }, + { + m: "test_histogram2_seconds_bucket\xff__type__\xffhistogram\xff__unit__\xffseconds\xffle\xff-20.0", + v: 2, + lset: labels.FromStrings( + "__name__", "test_histogram2_seconds_bucket", + "__type__", "histogram", + "__unit__", "seconds", + "le", "-20.0", + ), + }, + { + m: "test_histogram2_seconds_bucket\xff__type__\xffhistogram\xff__unit__\xffseconds\xffle\xff20.0", + v: 4, + lset: labels.FromStrings( + "__name__", "test_histogram2_seconds_bucket", + "__type__", "histogram", + "__unit__", "seconds", + "le", "20.0", + ), + es: []exemplar.Exemplar{ + {Labels: labels.FromStrings("dummyID", "59727"), Value: 15, HasTs: true, Ts: 1625851153146}, + }, + }, + { + m: "test_histogram2_seconds_bucket\xff__type__\xffhistogram\xff__unit__\xffseconds\xffle\xff30.0", + v: 6, + lset: labels.FromStrings( + "__name__", "test_histogram2_seconds_bucket", + "__type__", "histogram", + "__unit__", "seconds", + "le", "30.0", + ), + es: []exemplar.Exemplar{ + {Labels: labels.FromStrings("dummyID", "5617"), Value: 25, HasTs: true, Ts: 1625851153146}, + }, + }, + { + m: "test_histogram2_seconds_bucket\xff__type__\xffhistogram\xff__unit__\xffseconds\xffle\xff+Inf", + v: 6, + lset: labels.FromStrings( + "__name__", "test_histogram2_seconds_bucket", + "__type__", "histogram", + "__unit__", "seconds", + "le", "+Inf", + ), + }, + { + m: "test_histogram2_seconds\xff__type__\xffhistogram\xff__unit__\xffseconds", + shs: &histogram.Histogram{ + CounterResetHint: histogram.UnknownCounterReset, + Count: 6, + Sum: 50, + Schema: -53, + PositiveSpans: []histogram.Span{ + {Length: 3}, + }, + NegativeSpans: []histogram.Span{}, + PositiveBuckets: []int64{2, 0, 0}, + CustomValues: []float64{-20, 20, 30}, + }, + lset: labels.FromStrings( + "__name__", "test_histogram2_seconds", + "__type__", "histogram", + "__unit__", "seconds", + ), + es: []exemplar.Exemplar{ + {Labels: labels.FromStrings("dummyID", "59727"), Value: 15, HasTs: true, Ts: 1625851153146}, + {Labels: labels.FromStrings("dummyID", "5617"), Value: 25, HasTs: true, Ts: 1625851153146}, + }, + }, + { + m: "test_histogram_family", + help: "Test histogram metric family with two very simple histograms.", + }, + { + m: "test_histogram_family", + typ: model.MetricTypeHistogram, + }, + { + m: "test_histogram_family_count\xff__type__\xffhistogram\xfffoo\xffbar", + v: 5, + lset: labels.FromStrings( + "__name__", "test_histogram_family_count", + "__type__", "histogram", + "foo", "bar", + ), + }, + { + m: "test_histogram_family_sum\xff__type__\xffhistogram\xfffoo\xffbar", + v: 12.1, + lset: labels.FromStrings( + "__name__", "test_histogram_family_sum", + "__type__", "histogram", + "foo", "bar", + ), + }, + { + m: "test_histogram_family_bucket\xff__type__\xffhistogram\xfffoo\xffbar\xffle\xff1.1", + v: 2, + lset: labels.FromStrings( + "__name__", "test_histogram_family_bucket", + "__type__", "histogram", + "foo", "bar", + "le", "1.1", + ), + }, + { + m: "test_histogram_family_bucket\xff__type__\xffhistogram\xfffoo\xffbar\xffle\xff2.2", + v: 3, + lset: labels.FromStrings( + "__name__", "test_histogram_family_bucket", + "__type__", "histogram", + "foo", "bar", + "le", "2.2", + ), + }, + { + m: "test_histogram_family_bucket\xff__type__\xffhistogram\xfffoo\xffbar\xffle\xff+Inf", + v: 5, + lset: labels.FromStrings( + "__name__", "test_histogram_family_bucket", + "__type__", "histogram", + "foo", "bar", + "le", "+Inf", + ), + }, + { + m: "test_histogram_family\xff__type__\xffhistogram\xfffoo\xffbar", + shs: &histogram.Histogram{ + CounterResetHint: histogram.UnknownCounterReset, + Count: 5, + Sum: 12.1, + Schema: -53, + PositiveSpans: []histogram.Span{ + {Length: 3}, + }, + NegativeSpans: []histogram.Span{}, + PositiveBuckets: []int64{2, -1, 1}, + CustomValues: []float64{1.1, 2.2}, + }, + lset: labels.FromStrings( + "__name__", "test_histogram_family", + "__type__", "histogram", + "foo", "bar", + ), + }, + { + m: "test_histogram_family_count\xff__type__\xffhistogram\xfffoo\xffbaz", + v: 6, + lset: labels.FromStrings( + "__name__", "test_histogram_family_count", + "__type__", "histogram", + "foo", "baz", + ), + }, + { + m: "test_histogram_family_sum\xff__type__\xffhistogram\xfffoo\xffbaz", + v: 13.1, + lset: labels.FromStrings( + "__name__", "test_histogram_family_sum", + "__type__", "histogram", + "foo", "baz", + ), + }, + { + m: "test_histogram_family_bucket\xff__type__\xffhistogram\xfffoo\xffbaz\xffle\xff1.1", + v: 0, + lset: labels.FromStrings( + "__name__", "test_histogram_family_bucket", + "__type__", "histogram", + "foo", "baz", + "le", "1.1", + ), + }, + { + m: "test_histogram_family_bucket\xff__type__\xffhistogram\xfffoo\xffbaz\xffle\xff2.2", + v: 5, + lset: labels.FromStrings( + "__name__", "test_histogram_family_bucket", + "__type__", "histogram", + "foo", "baz", + "le", "2.2", + ), + }, + { + m: "test_histogram_family_bucket\xff__type__\xffhistogram\xfffoo\xffbaz\xffle\xff+Inf", + v: 6, + lset: labels.FromStrings( + "__name__", "test_histogram_family_bucket", + "__type__", "histogram", + "foo", "baz", + "le", "+Inf", + ), + }, + { + m: "test_histogram_family\xff__type__\xffhistogram\xfffoo\xffbaz", + shs: &histogram.Histogram{ + CounterResetHint: histogram.UnknownCounterReset, + Count: 6, + Sum: 13.1, + Schema: -53, + PositiveSpans: []histogram.Span{ + {Offset: 1, Length: 2}, + }, + NegativeSpans: []histogram.Span{}, + PositiveBuckets: []int64{5, -4}, + CustomValues: []float64{1.1, 2.2}, + }, + lset: labels.FromStrings( + "__name__", "test_histogram_family", + "__type__", "histogram", + "foo", "baz", + ), + }, + { + m: "empty_histogram", + help: "A histogram without observations and with a zero threshold of zero but with a no-op span to identify it as a native histogram.", + }, + { + m: "empty_histogram", + typ: model.MetricTypeHistogram, + }, + { + m: "empty_histogram\xff__type__\xffhistogram", + shs: &histogram.Histogram{ + CounterResetHint: histogram.UnknownCounterReset, + PositiveSpans: []histogram.Span{}, + NegativeSpans: []histogram.Span{}, + }, + lset: labels.FromStrings( + "__name__", "empty_histogram", + "__type__", "histogram", + ), + }, + }, + }, + } + + for _, tc := range testCases { + name := fmt.Sprintf("keepClassic=%v,typeAndUnit=%v", tc.keepClassic, tc.typeAndUnit) + t.Run(name, func(t *testing.T) { + p := NewProtobufParser(data, tc.keepClassic, true, tc.typeAndUnit, labels.NewSymbolTable()) + got := testParse(t, p) + requireEntries(t, tc.expected, got) + }) + } +} diff --git a/web/federate_test.go b/web/federate_test.go index 97aa45edba..e0845d7cd4 100644 --- a/web/federate_test.go +++ b/web/federate_test.go @@ -392,7 +392,7 @@ func TestFederationWithNativeHistograms(t *testing.T) { require.Equal(t, http.StatusOK, res.Code) body, err := io.ReadAll(res.Body) require.NoError(t, err) - p := textparse.NewProtobufParser(body, false, false, labels.NewSymbolTable()) + p := textparse.NewProtobufParser(body, false, false, false, labels.NewSymbolTable()) var actVec promql.Vector metricFamilies := 0 l := labels.Labels{}