refactor(convert): updated tests and moved formatOpenMetricsFloat

Signed-off-by: Naman-B-Parlecha <namanparlecha@gmail.com>
This commit is contained in:
Naman-B-Parlecha 2025-10-06 22:56:45 +05:30
parent ed67a0cbf1
commit 083d0fa835
5 changed files with 183 additions and 152 deletions

View File

@ -15,27 +15,22 @@ package histogram
import ( import (
"errors" "errors"
"fmt"
"math" "math"
"github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/labels"
) )
// BucketEmitter is a callback function type for emitting histogram bucket series.
// Used in remote write to append converted bucket time series.
type BucketEmitter func(labels labels.Labels, value float64) error
// ConvertNHCBToClassicHistogram converts Native Histogram Custom Buckets (NHCB) to classic histogram series. // ConvertNHCBToClassicHistogram converts Native Histogram Custom Buckets (NHCB) to classic histogram series.
// This conversion is needed in various scenarios where users need to get NHCB back to classic histogram format, // This conversion is needed in various scenarios where users need to get NHCB back to classic histogram format,
// such as Remote Write v1 for external system compatibility and migration use cases. // such as Remote Write v1 for external system compatibility and migration use cases.
func ConvertNHCBToClassicHistogram(nhcb any, labels labels.Labels, lblBuilder *labels.Builder, bucketSeries BucketEmitter) error { func ConvertNHCBToClassic(nhcb any, lset labels.Labels, lsetBuilder *labels.Builder, emitSeriesFn func(labels labels.Labels, value float64) error) error {
baseName := labels.Get("__name__") baseName := lset.Get("__name__")
if baseName == "" { if baseName == "" {
return errors.New("metric name label '__name__' is missing") return errors.New("metric name label '__name__' is missing")
} }
oldLabels := lblBuilder.Labels() oldLabels := lsetBuilder.Labels()
defer lblBuilder.Reset(oldLabels) defer lsetBuilder.Reset(oldLabels)
var ( var (
customValues []float64 customValues []float64
@ -45,6 +40,9 @@ func ConvertNHCBToClassicHistogram(nhcb any, labels labels.Labels, lblBuilder *l
switch h := nhcb.(type) { switch h := nhcb.(type) {
case *Histogram: case *Histogram:
if h.Schema != -53 {
return errors.New("unsupported histogram schema, only NHCB converstion is supported")
}
customValues = h.CustomValues customValues = h.CustomValues
positiveBuckets = make([]float64, len(h.PositiveBuckets)) positiveBuckets = make([]float64, len(h.PositiveBuckets))
for i, v := range h.PositiveBuckets { for i, v := range h.PositiveBuckets {
@ -57,6 +55,9 @@ func ConvertNHCBToClassicHistogram(nhcb any, labels labels.Labels, lblBuilder *l
count = float64(h.Count) count = float64(h.Count)
sum = h.Sum sum = h.Sum
case *FloatHistogram: case *FloatHistogram:
if h.Schema != -53 {
return errors.New("unsupported histogram schema, only NHCB converstion is supported")
}
customValues = h.CustomValues customValues = h.CustomValues
positiveBuckets = h.PositiveBuckets positiveBuckets = h.PositiveBuckets
count = h.Count count = h.Count
@ -75,34 +76,30 @@ func ConvertNHCBToClassicHistogram(nhcb any, labels labels.Labels, lblBuilder *l
currCount := float64(0) currCount := float64(0)
for i := range customValues { for i := range customValues {
currCount = positiveBuckets[i] currCount = positiveBuckets[i]
lblBuilder.Reset(labels) lsetBuilder.Reset(lset)
lblBuilder.Set("__name__", baseName+"_bucket") lsetBuilder.Set("__name__", baseName+"_bucket")
lblBuilder.Set("le", fmt.Sprintf("%g", customValues[i])) lsetBuilder.Set("le", labels.FormatOpenMetricsFloat(customValues[i]))
bucketLabels := lblBuilder.Labels() if err := emitSeriesFn(lsetBuilder.Labels(), currCount); err != nil {
if err := bucketSeries(bucketLabels, currCount); err != nil {
return err return err
} }
} }
lblBuilder.Reset(labels) lsetBuilder.Reset(lset)
lblBuilder.Set("__name__", baseName+"_bucket") lsetBuilder.Set("__name__", baseName+"_bucket")
lblBuilder.Set("le", fmt.Sprintf("%g", math.Inf(1))) lsetBuilder.Set("le", labels.FormatOpenMetricsFloat(math.Inf(1)))
infBucketLabels := lblBuilder.Labels() if err := emitSeriesFn(lsetBuilder.Labels(), currCount); err != nil {
if err := bucketSeries(infBucketLabels, currCount); err != nil {
return err return err
} }
lblBuilder.Reset(labels) lsetBuilder.Reset(lset)
lblBuilder.Set("__name__", baseName+"_count") lsetBuilder.Set("__name__", baseName+"_count")
countLabels := lblBuilder.Labels() if err := emitSeriesFn(lsetBuilder.Labels(), count); err != nil {
if err := bucketSeries(countLabels, count); err != nil {
return err return err
} }
lblBuilder.Reset(labels) lsetBuilder.Reset(lset)
lblBuilder.Set("__name__", baseName+"_sum") lsetBuilder.Set("__name__", baseName+"_sum")
sumLabels := lblBuilder.Labels() if err := emitSeriesFn(lsetBuilder.Labels(), sum); err != nil {
if err := bucketSeries(sumLabels, sum); err != nil {
return err return err
} }

View File

@ -1,11 +1,11 @@
// Copyright 2025 The Prometheus Authors // Copyright 2025 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License"); // Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. // you may not use this filset elabels.FromStrings("__name__", "test_metric", "le",)xcept in compliance with the License.
// You may obtain a copy of the License at // You may obtain a copy of the License at
// //
// http://www.apache.org/licenses/LICENSE-2.0 // http://www.apache.org/licenses/LICENSE-2.0
// //
// Unless required by applicable law or agreed to in writing, software // Unlsetsslabels.FromStrings("__name__", "test_metric", "le",) required by applicablset llabels.FromStrings("__name__", "test_metric", "le",)aw or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, // distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
@ -14,7 +14,6 @@
package histogram package histogram
import ( import (
"errors"
"testing" "testing"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@ -22,15 +21,9 @@ import (
"github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/labels"
) )
type ExpectedBucket struct { type sample struct {
le string lset labels.Labels
val float64 val float64
}
type ExpectedClassicHistogram struct {
buckets []ExpectedBucket
count float64
sum float64
} }
func TestConvertNHCBToClassicHistogram(t *testing.T) { func TestConvertNHCBToClassicHistogram(t *testing.T) {
@ -39,111 +32,109 @@ func TestConvertNHCBToClassicHistogram(t *testing.T) {
nhcb any nhcb any
labels labels.Labels labels labels.Labels
expectErr bool expectErr bool
expected ExpectedClassicHistogram expected []sample
}{ }{
{ {
name: "Valid Histogram", name: "valid histogram",
nhcb: &Histogram{ nhcb: &Histogram{
CustomValues: []float64{1, 2, 3}, CustomValues: []float64{1, 2, 3},
PositiveBuckets: []int64{10, 20, 30}, PositiveBuckets: []int64{10, 20, 30},
Count: 60, Count: 60,
Sum: 100.0, Sum: 100.0,
Schema: -53,
}, },
labels: labels.FromStrings("__name__", "test_metric"), labels: labels.FromStrings("__name__", "test_metric"),
expected: ExpectedClassicHistogram{ expected: []sample{
buckets: []ExpectedBucket{ {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "1.0"), val: 10},
{le: "1", val: 10}, {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "2.0"), val: 30},
{le: "2", val: 30}, {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "3.0"), val: 60},
{le: "3", val: 60}, {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "+Inf"), val: 60},
{le: "+Inf", val: 60}, {lset: labels.FromStrings("__name__", "test_metric_count"), val: 60},
}, {lset: labels.FromStrings("__name__", "test_metric_sum"), val: 100},
count: 60,
sum: 100,
}, },
}, },
{ {
name: "Valid FloatHistogram", name: "valid floatHistogram",
nhcb: &FloatHistogram{ nhcb: &FloatHistogram{
CustomValues: []float64{1, 2, 3}, CustomValues: []float64{1, 2, 3},
PositiveBuckets: []float64{20.0, 40.0, 60.0}, PositiveBuckets: []float64{20.0, 40.0, 60.0},
Count: 60.0, Count: 60.0,
Sum: 100.0, Sum: 100.0,
Schema: -53,
}, },
labels: labels.FromStrings("__name__", "test_metric"), labels: labels.FromStrings("__name__", "test_metric"),
expected: ExpectedClassicHistogram{ expected: []sample{
buckets: []ExpectedBucket{ {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "1.0"), val: 20},
{le: "1", val: 20}, {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "2.0"), val: 40},
{le: "2", val: 40}, {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "3.0"), val: 60},
{le: "3", val: 60}, {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "+Inf"), val: 60},
{le: "+Inf", val: 60}, {lset: labels.FromStrings("__name__", "test_metric_count"), val: 60},
}, {lset: labels.FromStrings("__name__", "test_metric_sum"), val: 100},
count: 60,
sum: 100,
}, },
}, },
{ {
name: "Empty Histogram", name: "empty histogram",
nhcb: &Histogram{ nhcb: &Histogram{
CustomValues: []float64{}, CustomValues: []float64{},
PositiveBuckets: []int64{}, PositiveBuckets: []int64{},
Count: 0, Count: 0,
Sum: 0.0, Sum: 0.0,
Schema: -53,
}, },
labels: labels.FromStrings("__name__", "test_metric"), labels: labels.FromStrings("__name__", "test_metric"),
expected: ExpectedClassicHistogram{ expected: []sample{
buckets: []ExpectedBucket{ {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "+Inf"), val: 0},
{le: "+Inf", val: 0}, {lset: labels.FromStrings("__name__", "test_metric_count"), val: 0},
}, {lset: labels.FromStrings("__name__", "test_metric_sum"), val: 0},
count: 0,
sum: 0,
}, },
}, },
{ {
name: "Missing __name__ label", name: "missing __name__ label",
nhcb: &Histogram{ nhcb: &Histogram{
CustomValues: []float64{1, 2, 3}, CustomValues: []float64{1, 2, 3},
PositiveBuckets: []int64{10, 20, 30}, PositiveBuckets: []int64{10, 20, 30},
Count: 60, Count: 60,
Sum: 100.0, Sum: 100.0,
Schema: -53,
}, },
labels: labels.FromStrings("job", "test_job"), labels: labels.FromStrings("job", "test_job"),
expectErr: true, expectErr: true,
}, },
{ {
name: "Unsupported histogram type", name: "unsupported histogram type",
nhcb: nil, nhcb: nil,
labels: labels.FromStrings("__name__", "test_metric"), labels: labels.FromStrings("__name__", "test_metric"),
expectErr: true, expectErr: true,
}, },
{ {
name: "Histogram with zero bucket counts", name: "histogram with zero bucket counts",
nhcb: &Histogram{ nhcb: &Histogram{
CustomValues: []float64{1, 2, 3}, CustomValues: []float64{1, 2, 3},
PositiveBuckets: []int64{0, 10, 0}, PositiveBuckets: []int64{0, 10, 0},
Count: 10, Count: 10,
Sum: 50.0, Sum: 50.0,
Schema: -53,
}, },
labels: labels.FromStrings("__name__", "test_metric"), labels: labels.FromStrings("__name__", "test_metric"),
expected: ExpectedClassicHistogram{ expected: []sample{
buckets: []ExpectedBucket{ {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "1.0"), val: 0},
{le: "1", val: 0}, {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "2.0"), val: 10},
{le: "2", val: 10}, {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "3.0"), val: 10},
{le: "3", val: 10}, {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "+Inf"), val: 10},
{le: "+Inf", val: 10}, {lset: labels.FromStrings("__name__", "test_metric_count"), val: 10},
}, {lset: labels.FromStrings("__name__", "test_metric_sum"), val: 50},
count: 10,
sum: 50,
}, },
}, },
{ {
name: "Mismatched bucket lengths", name: "mismatched bucket lengths",
nhcb: &Histogram{ nhcb: &Histogram{
CustomValues: []float64{1, 2}, CustomValues: []float64{1, 2},
PositiveBuckets: []int64{10, 20, 30}, PositiveBuckets: []int64{10, 20, 30},
Count: 60, Count: 60,
Sum: 100.0, Sum: 100.0,
Schema: -53,
}, },
labels: labels.FromStrings("__name__", "test_metric"), labels: labels.FromStrings("__name__", "test_metric_bucket"),
expectErr: true, expectErr: true,
}, },
{ {
@ -153,47 +144,71 @@ func TestConvertNHCBToClassicHistogram(t *testing.T) {
PositiveBuckets: []int64{10}, PositiveBuckets: []int64{10},
Count: 10, Count: 10,
Sum: 20.0, Sum: 20.0,
Schema: -53,
}, },
labels: labels.FromStrings("__name__", "test_metric"), labels: labels.FromStrings("__name__", "test_metric"),
expected: ExpectedClassicHistogram{ expected: []sample{
buckets: []ExpectedBucket{ {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "1.0"), val: 10},
{le: "1", val: 10}, {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "+Inf"), val: 10},
{le: "+Inf", val: 10}, {lset: labels.FromStrings("__name__", "test_metric_count"), val: 10},
}, {lset: labels.FromStrings("__name__", "test_metric_sum"), val: 20},
count: 10,
sum: 20,
}, },
}, },
{
name: "multiset label histogram",
nhcb: &Histogram{
CustomValues: []float64{1},
PositiveBuckets: []int64{10},
Count: 10,
Sum: 20.0,
Schema: -53,
},
labels: labels.FromStrings("__name__", "test_metric", "job", "test_job", "instance", "localhost:9090"),
expected: []sample{
{lset: labels.FromStrings("__name__", "test_metric_bucket", "job", "test_job", "instance", "localhost:9090", "le", "1.0"), val: 10},
{lset: labels.FromStrings("__name__", "test_metric_bucket", "job", "test_job", "instance", "localhost:9090", "le", "+Inf"), val: 10},
{lset: labels.FromStrings("__name__", "test_metric_count", "job", "test_job", "instance", "localhost:9090"), val: 10},
{lset: labels.FromStrings("__name__", "test_metric_sum", "job", "test_job", "instance", "localhost:9090"), val: 20},
},
},
{
name: "exponential histogram",
nhcb: &FloatHistogram{
Schema: 1,
ZeroThreshold: 0.01,
ZeroCount: 5.5,
Count: 3493.3,
Sum: 2349209.324,
PositiveSpans: []Span{
{-2, 1},
{2, 3},
},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1},
NegativeSpans: []Span{
{3, 2},
{3, 2},
},
NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000},
},
labels: labels.FromStrings("__name__", "test_metric_bucket"),
expectErr: true,
},
} }
labelBuilder := labels.NewBuilder(labels.EmptyLabels()) labelBuilder := labels.NewBuilder(labels.EmptyLabels())
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
var got ExpectedClassicHistogram var emittedSamples []sample
err := ConvertNHCBToClassicHistogram(tt.nhcb, tt.labels, labelBuilder, func(lbls labels.Labels, val float64) error { err := ConvertNHCBToClassic(tt.nhcb, tt.labels, labelBuilder, func(lbls labels.Labels, val float64) error {
switch lbls.Get("__name__") { emittedSamples = append(emittedSamples, sample{lset: lbls, val: val})
case tt.labels.Get("__name__") + "_bucket":
got.buckets = append(got.buckets, ExpectedBucket{
le: lbls.Get("le"),
val: val,
})
case tt.labels.Get("__name__") + "_count":
got.count = val
case tt.labels.Get("__name__") + "_sum":
got.sum = val
default:
return errors.New("unexpected metric name")
}
return nil return nil
}) })
require.Equal(t, tt.expectErr, err != nil, "unexpected error: %v", err) require.Equal(t, tt.expectErr, err != nil, "unexpected error: %v", err)
if !tt.expectErr { if !tt.expectErr {
require.Len(t, got.buckets, len(tt.expected.buckets)) require.Len(t, emittedSamples, len(tt.expected))
for i, expBucket := range tt.expected.buckets { for i, expSample := range tt.expected {
require.Equal(t, expBucket, got.buckets[i]) require.Equal(t, expSample, emittedSamples[i])
} }
require.Equal(t, tt.expected.count, got.count)
require.Equal(t, tt.expected.sum, got.sum)
} }
}) })
} }

60
model/labels/float.go Normal file
View File

@ -0,0 +1,60 @@
// Copyright 2025 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package labels
import (
"bytes"
"math"
"strconv"
"sync"
)
// floatFormatBufPool is exclusively used in formatOpenMetricsFloat.
var floatFormatBufPool = sync.Pool{
New: func() any {
// To contain at most 17 digits and additional syntax for a float64.
b := make([]byte, 0, 24)
return &b
},
}
// formatOpenMetricsFloat works like the usual Go string formatting of a float
// but appends ".0" if the resulting number would otherwise contain neither a
// "." nor an "e".
func FormatOpenMetricsFloat(f float64) string {
// A few common cases hardcoded.
switch {
case f == 1:
return "1.0"
case f == 0:
return "0.0"
case f == -1:
return "-1.0"
case math.IsNaN(f):
return "NaN"
case math.IsInf(f, +1):
return "+Inf"
case math.IsInf(f, -1):
return "-Inf"
}
bp := floatFormatBufPool.Get().(*[]byte)
defer floatFormatBufPool.Put(bp)
*bp = strconv.AppendFloat((*bp)[:0], f, 'g', -1, 64)
if bytes.ContainsAny(*bp, "e.") {
return string(*bp)
}
*bp = append(*bp, '.', '0')
return string(*bp)
}

View File

@ -773,7 +773,7 @@ func normalizeFloatsInLabelValues(t model.MetricType, l, v string) string {
if (t == model.MetricTypeSummary && l == model.QuantileLabel) || (t == model.MetricTypeHistogram && l == model.BucketLabel) { if (t == model.MetricTypeSummary && l == model.QuantileLabel) || (t == model.MetricTypeHistogram && l == model.BucketLabel) {
f, err := strconv.ParseFloat(v, 64) f, err := strconv.ParseFloat(v, 64)
if err == nil { if err == nil {
return formatOpenMetricsFloat(f) return labels.FormatOpenMetricsFloat(f)
} }
} }
return v return v

View File

@ -19,9 +19,7 @@ import (
"fmt" "fmt"
"io" "io"
"math" "math"
"strconv"
"strings" "strings"
"sync"
"unicode/utf8" "unicode/utf8"
"github.com/gogo/protobuf/types" "github.com/gogo/protobuf/types"
@ -34,15 +32,6 @@ import (
"github.com/prometheus/prometheus/schema" "github.com/prometheus/prometheus/schema"
) )
// floatFormatBufPool is exclusively used in formatOpenMetricsFloat.
var floatFormatBufPool = sync.Pool{
New: func() any {
// To contain at most 17 digits and additional syntax for a float64.
b := make([]byte, 0, 24)
return &b
},
}
// ProtobufParser parses the old Prometheus protobuf format and present it // ProtobufParser parses the old Prometheus protobuf format and present it
// as the text-style textparse.Parser interface. // as the text-style textparse.Parser interface.
// //
@ -632,7 +621,7 @@ func (p *ProtobufParser) getMagicLabel() (bool, string, string) {
qq := p.dec.GetSummary().GetQuantile() qq := p.dec.GetSummary().GetQuantile()
q := qq[p.fieldPos] q := qq[p.fieldPos]
p.fieldsDone = p.fieldPos == len(qq)-1 p.fieldsDone = p.fieldPos == len(qq)-1
return true, model.QuantileLabel, formatOpenMetricsFloat(q.GetQuantile()) return true, model.QuantileLabel, labels.FormatOpenMetricsFloat(q.GetQuantile())
case dto.MetricType_HISTOGRAM, dto.MetricType_GAUGE_HISTOGRAM: case dto.MetricType_HISTOGRAM, dto.MetricType_GAUGE_HISTOGRAM:
bb := p.dec.GetHistogram().GetBucket() bb := p.dec.GetHistogram().GetBucket()
if p.fieldPos >= len(bb) { if p.fieldPos >= len(bb) {
@ -641,41 +630,11 @@ func (p *ProtobufParser) getMagicLabel() (bool, string, string) {
} }
b := bb[p.fieldPos] b := bb[p.fieldPos]
p.fieldsDone = math.IsInf(b.GetUpperBound(), +1) p.fieldsDone = math.IsInf(b.GetUpperBound(), +1)
return true, model.BucketLabel, formatOpenMetricsFloat(b.GetUpperBound()) return true, model.BucketLabel, labels.FormatOpenMetricsFloat(b.GetUpperBound())
} }
return false, "", "" return false, "", ""
} }
// formatOpenMetricsFloat works like the usual Go string formatting of a float
// but appends ".0" if the resulting number would otherwise contain neither a
// "." nor an "e".
func formatOpenMetricsFloat(f float64) string {
// A few common cases hardcoded.
switch {
case f == 1:
return "1.0"
case f == 0:
return "0.0"
case f == -1:
return "-1.0"
case math.IsNaN(f):
return "NaN"
case math.IsInf(f, +1):
return "+Inf"
case math.IsInf(f, -1):
return "-Inf"
}
bp := floatFormatBufPool.Get().(*[]byte)
defer floatFormatBufPool.Put(bp)
*bp = strconv.AppendFloat((*bp)[:0], f, 'g', -1, 64)
if bytes.ContainsAny(*bp, "e.") {
return string(*bp)
}
*bp = append(*bp, '.', '0')
return string(*bp)
}
// isNativeHistogram returns false iff the provided histograms has no spans at // isNativeHistogram returns false iff the provided histograms has no spans at
// all (neither positive nor negative) and a zero threshold of 0 and a zero // all (neither positive nor negative) and a zero threshold of 0 and a zero
// count of 0. In principle, this could still be meant to be a native histogram // count of 0. In principle, this could still be meant to be a native histogram