diff --git a/model/histogram/convert.go b/model/histogram/convert.go index 847baea494..2ac23d1d6f 100644 --- a/model/histogram/convert.go +++ b/model/histogram/convert.go @@ -15,27 +15,22 @@ package histogram import ( "errors" - "fmt" "math" "github.com/prometheus/prometheus/model/labels" ) -// BucketEmitter is a callback function type for emitting histogram bucket series. -// Used in remote write to append converted bucket time series. -type BucketEmitter func(labels labels.Labels, value float64) error - // ConvertNHCBToClassicHistogram converts Native Histogram Custom Buckets (NHCB) to classic histogram series. // This conversion is needed in various scenarios where users need to get NHCB back to classic histogram format, // such as Remote Write v1 for external system compatibility and migration use cases. -func ConvertNHCBToClassicHistogram(nhcb any, labels labels.Labels, lblBuilder *labels.Builder, bucketSeries BucketEmitter) error { - baseName := labels.Get("__name__") +func ConvertNHCBToClassic(nhcb any, lset labels.Labels, lsetBuilder *labels.Builder, emitSeriesFn func(labels labels.Labels, value float64) error) error { + baseName := lset.Get("__name__") if baseName == "" { return errors.New("metric name label '__name__' is missing") } - oldLabels := lblBuilder.Labels() - defer lblBuilder.Reset(oldLabels) + oldLabels := lsetBuilder.Labels() + defer lsetBuilder.Reset(oldLabels) var ( customValues []float64 @@ -45,6 +40,9 @@ func ConvertNHCBToClassicHistogram(nhcb any, labels labels.Labels, lblBuilder *l switch h := nhcb.(type) { case *Histogram: + if h.Schema != -53 { + return errors.New("unsupported histogram schema, only NHCB converstion is supported") + } customValues = h.CustomValues positiveBuckets = make([]float64, len(h.PositiveBuckets)) for i, v := range h.PositiveBuckets { @@ -57,6 +55,9 @@ func ConvertNHCBToClassicHistogram(nhcb any, labels labels.Labels, lblBuilder *l count = float64(h.Count) sum = h.Sum case *FloatHistogram: + if h.Schema != -53 { + return errors.New("unsupported histogram schema, only NHCB converstion is supported") + } customValues = h.CustomValues positiveBuckets = h.PositiveBuckets count = h.Count @@ -75,34 +76,30 @@ func ConvertNHCBToClassicHistogram(nhcb any, labels labels.Labels, lblBuilder *l currCount := float64(0) for i := range customValues { currCount = positiveBuckets[i] - lblBuilder.Reset(labels) - lblBuilder.Set("__name__", baseName+"_bucket") - lblBuilder.Set("le", fmt.Sprintf("%g", customValues[i])) - bucketLabels := lblBuilder.Labels() - if err := bucketSeries(bucketLabels, currCount); err != nil { + lsetBuilder.Reset(lset) + lsetBuilder.Set("__name__", baseName+"_bucket") + lsetBuilder.Set("le", labels.FormatOpenMetricsFloat(customValues[i])) + if err := emitSeriesFn(lsetBuilder.Labels(), currCount); err != nil { return err } } - lblBuilder.Reset(labels) - lblBuilder.Set("__name__", baseName+"_bucket") - lblBuilder.Set("le", fmt.Sprintf("%g", math.Inf(1))) - infBucketLabels := lblBuilder.Labels() - if err := bucketSeries(infBucketLabels, currCount); err != nil { + lsetBuilder.Reset(lset) + lsetBuilder.Set("__name__", baseName+"_bucket") + lsetBuilder.Set("le", labels.FormatOpenMetricsFloat(math.Inf(1))) + if err := emitSeriesFn(lsetBuilder.Labels(), currCount); err != nil { return err } - lblBuilder.Reset(labels) - lblBuilder.Set("__name__", baseName+"_count") - countLabels := lblBuilder.Labels() - if err := bucketSeries(countLabels, count); err != nil { + lsetBuilder.Reset(lset) + lsetBuilder.Set("__name__", baseName+"_count") + if err := emitSeriesFn(lsetBuilder.Labels(), count); err != nil { return err } - lblBuilder.Reset(labels) - lblBuilder.Set("__name__", baseName+"_sum") - sumLabels := lblBuilder.Labels() - if err := bucketSeries(sumLabels, sum); err != nil { + lsetBuilder.Reset(lset) + lsetBuilder.Set("__name__", baseName+"_sum") + if err := emitSeriesFn(lsetBuilder.Labels(), sum); err != nil { return err } diff --git a/model/histogram/convert_test.go b/model/histogram/convert_test.go index 74d86286dc..e1d7b9ec29 100644 --- a/model/histogram/convert_test.go +++ b/model/histogram/convert_test.go @@ -1,11 +1,11 @@ // Copyright 2025 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. +// you may not use this filset elabels.FromStrings("__name__", "test_metric", "le",)xcept in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software +// Unlsetsslabels.FromStrings("__name__", "test_metric", "le",) required by applicablset llabels.FromStrings("__name__", "test_metric", "le",)aw or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and @@ -14,7 +14,6 @@ package histogram import ( - "errors" "testing" "github.com/stretchr/testify/require" @@ -22,15 +21,9 @@ import ( "github.com/prometheus/prometheus/model/labels" ) -type ExpectedBucket struct { - le string - val float64 -} - -type ExpectedClassicHistogram struct { - buckets []ExpectedBucket - count float64 - sum float64 +type sample struct { + lset labels.Labels + val float64 } func TestConvertNHCBToClassicHistogram(t *testing.T) { @@ -39,111 +32,109 @@ func TestConvertNHCBToClassicHistogram(t *testing.T) { nhcb any labels labels.Labels expectErr bool - expected ExpectedClassicHistogram + expected []sample }{ { - name: "Valid Histogram", + name: "valid histogram", nhcb: &Histogram{ CustomValues: []float64{1, 2, 3}, PositiveBuckets: []int64{10, 20, 30}, Count: 60, Sum: 100.0, + Schema: -53, }, labels: labels.FromStrings("__name__", "test_metric"), - expected: ExpectedClassicHistogram{ - buckets: []ExpectedBucket{ - {le: "1", val: 10}, - {le: "2", val: 30}, - {le: "3", val: 60}, - {le: "+Inf", val: 60}, - }, - count: 60, - sum: 100, + expected: []sample{ + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "1.0"), val: 10}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "2.0"), val: 30}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "3.0"), val: 60}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "+Inf"), val: 60}, + {lset: labels.FromStrings("__name__", "test_metric_count"), val: 60}, + {lset: labels.FromStrings("__name__", "test_metric_sum"), val: 100}, }, }, { - name: "Valid FloatHistogram", + name: "valid floatHistogram", nhcb: &FloatHistogram{ CustomValues: []float64{1, 2, 3}, PositiveBuckets: []float64{20.0, 40.0, 60.0}, Count: 60.0, Sum: 100.0, + Schema: -53, }, labels: labels.FromStrings("__name__", "test_metric"), - expected: ExpectedClassicHistogram{ - buckets: []ExpectedBucket{ - {le: "1", val: 20}, - {le: "2", val: 40}, - {le: "3", val: 60}, - {le: "+Inf", val: 60}, - }, - count: 60, - sum: 100, + expected: []sample{ + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "1.0"), val: 20}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "2.0"), val: 40}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "3.0"), val: 60}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "+Inf"), val: 60}, + {lset: labels.FromStrings("__name__", "test_metric_count"), val: 60}, + {lset: labels.FromStrings("__name__", "test_metric_sum"), val: 100}, }, }, { - name: "Empty Histogram", + name: "empty histogram", nhcb: &Histogram{ CustomValues: []float64{}, PositiveBuckets: []int64{}, Count: 0, Sum: 0.0, + Schema: -53, }, labels: labels.FromStrings("__name__", "test_metric"), - expected: ExpectedClassicHistogram{ - buckets: []ExpectedBucket{ - {le: "+Inf", val: 0}, - }, - count: 0, - sum: 0, + expected: []sample{ + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "+Inf"), val: 0}, + {lset: labels.FromStrings("__name__", "test_metric_count"), val: 0}, + {lset: labels.FromStrings("__name__", "test_metric_sum"), val: 0}, }, }, { - name: "Missing __name__ label", + name: "missing __name__ label", nhcb: &Histogram{ CustomValues: []float64{1, 2, 3}, PositiveBuckets: []int64{10, 20, 30}, Count: 60, Sum: 100.0, + Schema: -53, }, labels: labels.FromStrings("job", "test_job"), expectErr: true, }, { - name: "Unsupported histogram type", + name: "unsupported histogram type", nhcb: nil, labels: labels.FromStrings("__name__", "test_metric"), expectErr: true, }, { - name: "Histogram with zero bucket counts", + name: "histogram with zero bucket counts", nhcb: &Histogram{ CustomValues: []float64{1, 2, 3}, PositiveBuckets: []int64{0, 10, 0}, Count: 10, Sum: 50.0, + Schema: -53, }, labels: labels.FromStrings("__name__", "test_metric"), - expected: ExpectedClassicHistogram{ - buckets: []ExpectedBucket{ - {le: "1", val: 0}, - {le: "2", val: 10}, - {le: "3", val: 10}, - {le: "+Inf", val: 10}, - }, - count: 10, - sum: 50, + expected: []sample{ + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "1.0"), val: 0}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "2.0"), val: 10}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "3.0"), val: 10}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "+Inf"), val: 10}, + {lset: labels.FromStrings("__name__", "test_metric_count"), val: 10}, + {lset: labels.FromStrings("__name__", "test_metric_sum"), val: 50}, }, }, { - name: "Mismatched bucket lengths", + name: "mismatched bucket lengths", nhcb: &Histogram{ CustomValues: []float64{1, 2}, PositiveBuckets: []int64{10, 20, 30}, Count: 60, Sum: 100.0, + Schema: -53, }, - labels: labels.FromStrings("__name__", "test_metric"), + labels: labels.FromStrings("__name__", "test_metric_bucket"), expectErr: true, }, { @@ -153,47 +144,71 @@ func TestConvertNHCBToClassicHistogram(t *testing.T) { PositiveBuckets: []int64{10}, Count: 10, Sum: 20.0, + Schema: -53, }, labels: labels.FromStrings("__name__", "test_metric"), - expected: ExpectedClassicHistogram{ - buckets: []ExpectedBucket{ - {le: "1", val: 10}, - {le: "+Inf", val: 10}, - }, - count: 10, - sum: 20, + expected: []sample{ + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "1.0"), val: 10}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "+Inf"), val: 10}, + {lset: labels.FromStrings("__name__", "test_metric_count"), val: 10}, + {lset: labels.FromStrings("__name__", "test_metric_sum"), val: 20}, }, }, + { + name: "multiset label histogram", + nhcb: &Histogram{ + CustomValues: []float64{1}, + PositiveBuckets: []int64{10}, + Count: 10, + Sum: 20.0, + Schema: -53, + }, + labels: labels.FromStrings("__name__", "test_metric", "job", "test_job", "instance", "localhost:9090"), + expected: []sample{ + {lset: labels.FromStrings("__name__", "test_metric_bucket", "job", "test_job", "instance", "localhost:9090", "le", "1.0"), val: 10}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "job", "test_job", "instance", "localhost:9090", "le", "+Inf"), val: 10}, + {lset: labels.FromStrings("__name__", "test_metric_count", "job", "test_job", "instance", "localhost:9090"), val: 10}, + {lset: labels.FromStrings("__name__", "test_metric_sum", "job", "test_job", "instance", "localhost:9090"), val: 20}, + }, + }, + { + name: "exponential histogram", + nhcb: &FloatHistogram{ + Schema: 1, + ZeroThreshold: 0.01, + ZeroCount: 5.5, + Count: 3493.3, + Sum: 2349209.324, + PositiveSpans: []Span{ + {-2, 1}, + {2, 3}, + }, + PositiveBuckets: []float64{1, 3.3, 4.2, 0.1}, + NegativeSpans: []Span{ + {3, 2}, + {3, 2}, + }, + NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000}, + }, + labels: labels.FromStrings("__name__", "test_metric_bucket"), + expectErr: true, + }, } labelBuilder := labels.NewBuilder(labels.EmptyLabels()) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - var got ExpectedClassicHistogram - err := ConvertNHCBToClassicHistogram(tt.nhcb, tt.labels, labelBuilder, func(lbls labels.Labels, val float64) error { - switch lbls.Get("__name__") { - case tt.labels.Get("__name__") + "_bucket": - got.buckets = append(got.buckets, ExpectedBucket{ - le: lbls.Get("le"), - val: val, - }) - case tt.labels.Get("__name__") + "_count": - got.count = val - case tt.labels.Get("__name__") + "_sum": - got.sum = val - default: - return errors.New("unexpected metric name") - } + var emittedSamples []sample + err := ConvertNHCBToClassic(tt.nhcb, tt.labels, labelBuilder, func(lbls labels.Labels, val float64) error { + emittedSamples = append(emittedSamples, sample{lset: lbls, val: val}) return nil }) require.Equal(t, tt.expectErr, err != nil, "unexpected error: %v", err) if !tt.expectErr { - require.Len(t, got.buckets, len(tt.expected.buckets)) - for i, expBucket := range tt.expected.buckets { - require.Equal(t, expBucket, got.buckets[i]) + require.Len(t, emittedSamples, len(tt.expected)) + for i, expSample := range tt.expected { + require.Equal(t, expSample, emittedSamples[i]) } - require.Equal(t, tt.expected.count, got.count) - require.Equal(t, tt.expected.sum, got.sum) } }) } diff --git a/model/labels/float.go b/model/labels/float.go new file mode 100644 index 0000000000..e052347cd3 --- /dev/null +++ b/model/labels/float.go @@ -0,0 +1,60 @@ +// Copyright 2025 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package labels + +import ( + "bytes" + "math" + "strconv" + "sync" +) + +// floatFormatBufPool is exclusively used in formatOpenMetricsFloat. +var floatFormatBufPool = sync.Pool{ + New: func() any { + // To contain at most 17 digits and additional syntax for a float64. + b := make([]byte, 0, 24) + return &b + }, +} + +// formatOpenMetricsFloat works like the usual Go string formatting of a float +// but appends ".0" if the resulting number would otherwise contain neither a +// "." nor an "e". +func FormatOpenMetricsFloat(f float64) string { + // A few common cases hardcoded. + switch { + case f == 1: + return "1.0" + case f == 0: + return "0.0" + case f == -1: + return "-1.0" + case math.IsNaN(f): + return "NaN" + case math.IsInf(f, +1): + return "+Inf" + case math.IsInf(f, -1): + return "-Inf" + } + bp := floatFormatBufPool.Get().(*[]byte) + defer floatFormatBufPool.Put(bp) + + *bp = strconv.AppendFloat((*bp)[:0], f, 'g', -1, 64) + if bytes.ContainsAny(*bp, "e.") { + return string(*bp) + } + *bp = append(*bp, '.', '0') + return string(*bp) +} diff --git a/model/textparse/openmetricsparse.go b/model/textparse/openmetricsparse.go index 4e592167f3..505e45fc40 100644 --- a/model/textparse/openmetricsparse.go +++ b/model/textparse/openmetricsparse.go @@ -773,7 +773,7 @@ func normalizeFloatsInLabelValues(t model.MetricType, l, v string) string { if (t == model.MetricTypeSummary && l == model.QuantileLabel) || (t == model.MetricTypeHistogram && l == model.BucketLabel) { f, err := strconv.ParseFloat(v, 64) if err == nil { - return formatOpenMetricsFloat(f) + return labels.FormatOpenMetricsFloat(f) } } return v diff --git a/model/textparse/protobufparse.go b/model/textparse/protobufparse.go index e7ce710491..80091bfd85 100644 --- a/model/textparse/protobufparse.go +++ b/model/textparse/protobufparse.go @@ -19,9 +19,7 @@ import ( "fmt" "io" "math" - "strconv" "strings" - "sync" "unicode/utf8" "github.com/gogo/protobuf/types" @@ -34,15 +32,6 @@ import ( "github.com/prometheus/prometheus/schema" ) -// floatFormatBufPool is exclusively used in formatOpenMetricsFloat. -var floatFormatBufPool = sync.Pool{ - New: func() any { - // To contain at most 17 digits and additional syntax for a float64. - b := make([]byte, 0, 24) - return &b - }, -} - // ProtobufParser parses the old Prometheus protobuf format and present it // as the text-style textparse.Parser interface. // @@ -632,7 +621,7 @@ func (p *ProtobufParser) getMagicLabel() (bool, string, string) { qq := p.dec.GetSummary().GetQuantile() q := qq[p.fieldPos] p.fieldsDone = p.fieldPos == len(qq)-1 - return true, model.QuantileLabel, formatOpenMetricsFloat(q.GetQuantile()) + return true, model.QuantileLabel, labels.FormatOpenMetricsFloat(q.GetQuantile()) case dto.MetricType_HISTOGRAM, dto.MetricType_GAUGE_HISTOGRAM: bb := p.dec.GetHistogram().GetBucket() if p.fieldPos >= len(bb) { @@ -641,41 +630,11 @@ func (p *ProtobufParser) getMagicLabel() (bool, string, string) { } b := bb[p.fieldPos] p.fieldsDone = math.IsInf(b.GetUpperBound(), +1) - return true, model.BucketLabel, formatOpenMetricsFloat(b.GetUpperBound()) + return true, model.BucketLabel, labels.FormatOpenMetricsFloat(b.GetUpperBound()) } return false, "", "" } -// formatOpenMetricsFloat works like the usual Go string formatting of a float -// but appends ".0" if the resulting number would otherwise contain neither a -// "." nor an "e". -func formatOpenMetricsFloat(f float64) string { - // A few common cases hardcoded. - switch { - case f == 1: - return "1.0" - case f == 0: - return "0.0" - case f == -1: - return "-1.0" - case math.IsNaN(f): - return "NaN" - case math.IsInf(f, +1): - return "+Inf" - case math.IsInf(f, -1): - return "-Inf" - } - bp := floatFormatBufPool.Get().(*[]byte) - defer floatFormatBufPool.Put(bp) - - *bp = strconv.AppendFloat((*bp)[:0], f, 'g', -1, 64) - if bytes.ContainsAny(*bp, "e.") { - return string(*bp) - } - *bp = append(*bp, '.', '0') - return string(*bp) -} - // isNativeHistogram returns false iff the provided histograms has no spans at // all (neither positive nor negative) and a zero threshold of 0 and a zero // count of 0. In principle, this could still be meant to be a native histogram