diff --git a/model/histogram/convert.go b/model/histogram/convert.go index 19376db93c..d43b8182d6 100644 --- a/model/histogram/convert.go +++ b/model/histogram/convert.go @@ -1,4 +1,4 @@ -// Copyright 2025 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -30,6 +30,9 @@ func ConvertNHCBToClassic(nhcb any, lset labels.Labels, lsetBuilder *labels.Buil return errors.New("metric name label '__name__' is missing") } + // We preserve original labels and restore them after conversion. + // This is to ensure that no modifications are made to the original labels + // that the queue_manager relies on. oldLabels := lsetBuilder.Labels() defer lsetBuilder.Reset(oldLabels) @@ -37,6 +40,8 @@ func ConvertNHCBToClassic(nhcb any, lset labels.Labels, lsetBuilder *labels.Buil customValues []float64 positiveBuckets []float64 count, sum float64 + idx int // This index is to track buckets in Classic Histogram + currIdx int // This index is to track buckets in Native Histogram ) switch h := nhcb.(type) { @@ -44,13 +49,35 @@ func ConvertNHCBToClassic(nhcb any, lset labels.Labels, lsetBuilder *labels.Buil if !IsCustomBucketsSchema(h.Schema) { return errors.New("unsupported histogram schema, not a NHCB") } + + filledBuckets := 0 + totalBuckets := 0 + for _, span := range h.PositiveSpans { + filledBuckets += int(span.Length) + totalBuckets += int(span.Offset) + int(span.Length) + } + if filledBuckets != len(h.PositiveBuckets) { + return errors.New("mismatched lengths of positive buckets and spans") + } + if totalBuckets > len(h.CustomValues) { + return errors.New("mismatched lengths of custom values and buckets from span calculation") + } + customValues = h.CustomValues - positiveBuckets = make([]float64, len(h.PositiveBuckets)) - for i, v := range h.PositiveBuckets { - if i == 0 { - positiveBuckets[i] = float64(v) - } else { - positiveBuckets[i] = float64(v) + positiveBuckets[i-1] + positiveBuckets = make([]float64, len(customValues)) + + // Histograms are in delta format so we first bring them to absolute format. + acc := int64(0) + for _, s := range h.PositiveSpans { + for i := 0; i < int(s.Offset); i++ { + positiveBuckets[idx] = float64(acc) + idx++ + } + for i := 0; i < int(s.Length); i++ { + acc += h.PositiveBuckets[currIdx] + positiveBuckets[idx] = float64(acc) + idx++ + currIdx++ } } count = float64(h.Count) @@ -59,8 +86,34 @@ func ConvertNHCBToClassic(nhcb any, lset labels.Labels, lsetBuilder *labels.Buil if !IsCustomBucketsSchema(h.Schema) { return errors.New("unsupported histogram schema, not a NHCB") } + + filledBuckets := 0 + totalBuckets := 0 + for _, s := range h.PositiveSpans { + filledBuckets += int(s.Length) + totalBuckets += int(s.Offset) + int(s.Length) + } + if filledBuckets != len(h.PositiveBuckets) { + return errors.New("mismatched lengths of positive buckets and spans") + } + if totalBuckets > len(h.CustomValues) { + return errors.New("mismatched lengths of custom values and buckets from span calculation") + } + customValues = h.CustomValues - positiveBuckets = h.PositiveBuckets + positiveBuckets = make([]float64, len(customValues)) + + for _, span := range h.PositiveSpans { + // Since Float Histogram is already in absolute format we should + // keep the sparse buckets empty so we jump and go to next filled + // bucket index. + idx += int(span.Offset) + for i := 0; i < int(span.Length); i++ { + positiveBuckets[idx] = h.PositiveBuckets[currIdx] + idx++ + currIdx++ + } + } count = h.Count sum = h.Sum default: @@ -75,11 +128,11 @@ func ConvertNHCBToClassic(nhcb any, lset labels.Labels, lsetBuilder *labels.Buil } currCount := float64(0) - for i := range customValues { - currCount = positiveBuckets[i] + for i, val := range customValues { + currCount += positiveBuckets[i] lsetBuilder.Reset(lset) lsetBuilder.Set("__name__", baseName+"_bucket") - lsetBuilder.Set("le", labels.FormatOpenMetricsFloat(customValues[i])) + lsetBuilder.Set("le", labels.FormatOpenMetricsFloat(val)) if err := emitSeriesFn(lsetBuilder.Labels(), currCount); err != nil { return err } diff --git a/model/histogram/convert_test.go b/model/histogram/convert_test.go index 561e963b95..16c50d6263 100644 --- a/model/histogram/convert_test.go +++ b/model/histogram/convert_test.go @@ -1,11 +1,11 @@ -// Copyright 2025 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this filset elabels.FromStrings("__name__", "test_metric", "le",)xcept in compliance with the License. +// you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // -// Unlsetsslabels.FromStrings("__name__", "test_metric", "le",) required by applicablset llabels.FromStrings("__name__", "test_metric", "le",)aw or agreed to in writing, software +// Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and @@ -39,17 +39,20 @@ func TestConvertNHCBToClassicHistogram(t *testing.T) { nhcb: &Histogram{ CustomValues: []float64{1, 2, 3}, PositiveBuckets: []int64{10, 20, 30}, - Count: 60, - Sum: 100.0, - Schema: -53, + PositiveSpans: []Span{ + {Offset: 0, Length: 3}, + }, + Count: 100, + Sum: 100.0, + Schema: CustomBucketsSchema, }, labels: labels.FromStrings("__name__", "test_metric"), expected: []sample{ {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "1.0"), val: 10}, - {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "2.0"), val: 30}, - {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "3.0"), val: 60}, - {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "+Inf"), val: 60}, - {lset: labels.FromStrings("__name__", "test_metric_count"), val: 60}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "2.0"), val: 40}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "3.0"), val: 100}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "+Inf"), val: 100}, + {lset: labels.FromStrings("__name__", "test_metric_count"), val: 100}, {lset: labels.FromStrings("__name__", "test_metric_sum"), val: 100}, }, }, @@ -57,18 +60,21 @@ func TestConvertNHCBToClassicHistogram(t *testing.T) { name: "valid floatHistogram", nhcb: &FloatHistogram{ CustomValues: []float64{1, 2, 3}, - PositiveBuckets: []float64{20.0, 40.0, 60.0}, - Count: 60.0, - Sum: 100.0, - Schema: -53, + PositiveBuckets: []float64{20.0, 40.0, 60.0}, // 20 -> 60 ->120 + PositiveSpans: []Span{ + {Offset: 0, Length: 3}, + }, + Count: 120.0, + Sum: 100.0, + Schema: CustomBucketsSchema, }, labels: labels.FromStrings("__name__", "test_metric"), expected: []sample{ {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "1.0"), val: 20}, - {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "2.0"), val: 40}, - {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "3.0"), val: 60}, - {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "+Inf"), val: 60}, - {lset: labels.FromStrings("__name__", "test_metric_count"), val: 60}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "2.0"), val: 60}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "3.0"), val: 120}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "+Inf"), val: 120}, + {lset: labels.FromStrings("__name__", "test_metric_count"), val: 120}, {lset: labels.FromStrings("__name__", "test_metric_sum"), val: 100}, }, }, @@ -77,9 +83,10 @@ func TestConvertNHCBToClassicHistogram(t *testing.T) { nhcb: &Histogram{ CustomValues: []float64{}, PositiveBuckets: []int64{}, + PositiveSpans: []Span{}, Count: 0, Sum: 0.0, - Schema: -53, + Schema: CustomBucketsSchema, }, labels: labels.FromStrings("__name__", "test_metric"), expected: []sample{ @@ -93,9 +100,9 @@ func TestConvertNHCBToClassicHistogram(t *testing.T) { nhcb: &Histogram{ CustomValues: []float64{1, 2, 3}, PositiveBuckets: []int64{10, 20, 30}, - Count: 60, + Count: 100, Sum: 100.0, - Schema: -53, + Schema: CustomBucketsSchema, }, labels: labels.FromStrings("job", "test_job"), expectErr: true, @@ -111,28 +118,45 @@ func TestConvertNHCBToClassicHistogram(t *testing.T) { nhcb: &Histogram{ CustomValues: []float64{1, 2, 3}, PositiveBuckets: []int64{0, 10, 0}, - Count: 10, - Sum: 50.0, - Schema: -53, + PositiveSpans: []Span{ + {Offset: 0, Length: 3}, + }, + Count: 20, + Sum: 50.0, + Schema: CustomBucketsSchema, }, labels: labels.FromStrings("__name__", "test_metric"), expected: []sample{ {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "1.0"), val: 0}, {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "2.0"), val: 10}, - {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "3.0"), val: 10}, - {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "+Inf"), val: 10}, - {lset: labels.FromStrings("__name__", "test_metric_count"), val: 10}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "3.0"), val: 20}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "+Inf"), val: 20}, + {lset: labels.FromStrings("__name__", "test_metric_count"), val: 20}, {lset: labels.FromStrings("__name__", "test_metric_sum"), val: 50}, }, }, { - name: "mismatched bucket lengths", + name: "mismatched bucket lengths with more filled bucket count", nhcb: &Histogram{ CustomValues: []float64{1, 2}, PositiveBuckets: []int64{10, 20, 30}, - Count: 60, + PositiveSpans: []Span{{Offset: 0, Length: 3}}, + Count: 100, Sum: 100.0, - Schema: -53, + Schema: CustomBucketsSchema, + }, + labels: labels.FromStrings("__name__", "test_metric_bucket"), + expectErr: true, + }, + { + name: "mismatched bucket lengths with less filled bucket count", + nhcb: &Histogram{ + CustomValues: []float64{1, 2}, + PositiveBuckets: []int64{10}, + PositiveSpans: []Span{{Offset: 0, Length: 2}}, + Count: 100, + Sum: 100.0, + Schema: CustomBucketsSchema, }, labels: labels.FromStrings("__name__", "test_metric_bucket"), expectErr: true, @@ -142,9 +166,12 @@ func TestConvertNHCBToClassicHistogram(t *testing.T) { nhcb: &Histogram{ CustomValues: []float64{1}, PositiveBuckets: []int64{10}, - Count: 10, - Sum: 20.0, - Schema: -53, + PositiveSpans: []Span{ + {Offset: 0, Length: 1}, + }, + Count: 10, + Sum: 20.0, + Schema: CustomBucketsSchema, }, labels: labels.FromStrings("__name__", "test_metric"), expected: []sample{ @@ -159,9 +186,12 @@ func TestConvertNHCBToClassicHistogram(t *testing.T) { nhcb: &Histogram{ CustomValues: []float64{1}, PositiveBuckets: []int64{10}, - Count: 10, - Sum: 20.0, - Schema: -53, + PositiveSpans: []Span{ + {Offset: 0, Length: 1}, + }, + Count: 10, + Sum: 20.0, + Schema: CustomBucketsSchema, }, labels: labels.FromStrings("__name__", "test_metric", "job", "test_job", "instance", "localhost:9090"), expected: []sample{ @@ -193,6 +223,68 @@ func TestConvertNHCBToClassicHistogram(t *testing.T) { labels: labels.FromStrings("__name__", "test_metric_bucket"), expectErr: true, }, + { + name: "sparse histogram", + nhcb: &Histogram{ + Schema: CustomBucketsSchema, + CustomValues: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + PositiveSpans: []Span{ + {0, 2}, + {4, 1}, + {1, 2}, + }, + PositiveBuckets: []int64{1, 2, 3, 4, 5}, // 1 -> 3 -> 3 -> 3 -> 3 -> 3 -> 6 ->6 ->10 -> 15 + Count: 53, // 1 -> 4 -> 7 -> 10 -> 13 -> 16 -> 22 -> 28 -> 38 -> 53 + Sum: 123, + }, + labels: labels.FromStrings("__name__", "test_metric"), + expected: []sample{ + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "1.0"), val: 1}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "2.0"), val: 4}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "3.0"), val: 7}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "4.0"), val: 10}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "5.0"), val: 13}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "6.0"), val: 16}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "7.0"), val: 22}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "8.0"), val: 28}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "9.0"), val: 38}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "10.0"), val: 53}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "+Inf"), val: 53}, + {lset: labels.FromStrings("__name__", "test_metric_count"), val: 53}, + {lset: labels.FromStrings("__name__", "test_metric_sum"), val: 123}, + }, + }, + { + name: "sparse float histogram", + nhcb: &FloatHistogram{ + Schema: CustomBucketsSchema, + CustomValues: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + PositiveSpans: []Span{ + {0, 2}, + {4, 1}, + {1, 2}, + }, + PositiveBuckets: []float64{1, 2, 3, 4, 5}, // 1 -> 2 -> 0 -> 0 -> 0 -> 0 -> 3 -> 0 -> 4 -> 5 + Count: 15, // 1 -> 3 -> 3 -> 3 -> 3 -> 3 -> 6 -> 6 -> 10 -> 15 + Sum: 123, + }, + labels: labels.FromStrings("__name__", "test_metric"), + expected: []sample{ + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "1.0"), val: 1}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "2.0"), val: 3}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "3.0"), val: 3}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "4.0"), val: 3}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "5.0"), val: 3}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "6.0"), val: 3}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "7.0"), val: 6}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "8.0"), val: 6}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "9.0"), val: 10}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "10.0"), val: 15}, + {lset: labels.FromStrings("__name__", "test_metric_bucket", "le", "+Inf"), val: 15}, + {lset: labels.FromStrings("__name__", "test_metric_count"), val: 15}, + {lset: labels.FromStrings("__name__", "test_metric_sum"), val: 123}, + }, + }, } labelBuilder := labels.NewBuilder(labels.EmptyLabels()) diff --git a/model/labels/float.go b/model/labels/float.go index 030ae9c0e0..c526a5b2a6 100644 --- a/model/labels/float.go +++ b/model/labels/float.go @@ -1,4 +1,4 @@ -// Copyright 2025 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at