diff --git a/model/histogram/float_histogram.go b/model/histogram/float_histogram.go index ded7dc400d..58f13c8cf3 100644 --- a/model/histogram/float_histogram.go +++ b/model/histogram/float_histogram.go @@ -34,8 +34,8 @@ type FloatHistogram struct { // They are all for base-2 bucket schemas, where 1 is a bucket boundary in // each case, and then each power of two is divided into 2^n logarithmic buckets. // Or in other words, each bucket boundary is the previous boundary times - // 2^(2^-n). Another valid schema number is 127 for custom buckets, defined by - // the CustomBounds field. + // 2^(2^-n). Another valid schema number is -53 for custom buckets, defined by + // the CustomValues field. Schema int32 // Width of the zero bucket. ZeroThreshold float64 @@ -53,9 +53,9 @@ type FloatHistogram struct { // Holds the custom (usually upper) bounds for bucket definitions, otherwise nil. // This slice is interned, to be treated as immutable and copied by reference. // These numbers should be strictly increasing. This field is only used when the - // schema is 127, and the ZeroThreshold, ZeroCount, NegativeSpans and NegativeBuckets - // fields are not used. - CustomBounds []float64 + // schema is for custom buckets, and the ZeroThreshold, ZeroCount, NegativeSpans + // and NegativeBuckets fields are not used. + CustomValues []float64 } func (h *FloatHistogram) UsesCustomBuckets() bool { @@ -72,7 +72,10 @@ func (h *FloatHistogram) Copy() *FloatHistogram { } if h.UsesCustomBuckets() { - c.CustomBounds = h.CustomBounds + if len(h.CustomValues) != 0 { + c.CustomValues = make([]float64, len(h.CustomValues)) + copy(c.CustomValues, h.CustomValues) + } } else { c.ZeroThreshold = h.ZeroThreshold c.ZeroCount = h.ZeroCount @@ -114,7 +117,8 @@ func (h *FloatHistogram) CopyTo(to *FloatHistogram) { to.NegativeSpans = clearIfNotNil(to.NegativeSpans) to.NegativeBuckets = clearIfNotNil(to.NegativeBuckets) - to.CustomBounds = h.CustomBounds + to.CustomValues = resize(to.CustomValues, len(h.CustomValues)) + copy(to.CustomValues, h.CustomValues) } else { to.ZeroThreshold = h.ZeroThreshold to.ZeroCount = h.ZeroCount @@ -125,7 +129,7 @@ func (h *FloatHistogram) CopyTo(to *FloatHistogram) { to.NegativeBuckets = resize(to.NegativeBuckets, len(h.NegativeBuckets)) copy(to.NegativeBuckets, h.NegativeBuckets) - to.CustomBounds = clearIfNotNil(to.CustomBounds) + to.CustomValues = clearIfNotNil(to.CustomValues) } to.PositiveSpans = resize(to.PositiveSpans, len(h.PositiveSpans)) @@ -311,7 +315,7 @@ func (h *FloatHistogram) Add(other *FloatHistogram) (*FloatHistogram, error) { if h.UsesCustomBuckets() != other.UsesCustomBuckets() { return nil, ErrHistogramsIncompatibleSchema } - if h.UsesCustomBuckets() && !floatBucketsMatch(h.CustomBounds, other.CustomBounds) { + if h.UsesCustomBuckets() && !FloatBucketsMatch(h.CustomValues, other.CustomValues) { return nil, ErrHistogramsIncompatibleBounds } @@ -387,7 +391,7 @@ func (h *FloatHistogram) Sub(other *FloatHistogram) (*FloatHistogram, error) { if h.UsesCustomBuckets() != other.UsesCustomBuckets() { return nil, ErrHistogramsIncompatibleSchema } - if h.UsesCustomBuckets() && !floatBucketsMatch(h.CustomBounds, other.CustomBounds) { + if h.UsesCustomBuckets() && !FloatBucketsMatch(h.CustomValues, other.CustomValues) { return nil, ErrHistogramsIncompatibleBounds } @@ -454,7 +458,7 @@ func (h *FloatHistogram) Equals(h2 *FloatHistogram) bool { } if h.UsesCustomBuckets() { - if !floatBucketsMatch(h.CustomBounds, h2.CustomBounds) { + if !FloatBucketsMatch(h.CustomValues, h2.CustomValues) { return false } } @@ -467,14 +471,14 @@ func (h *FloatHistogram) Equals(h2 *FloatHistogram) bool { if !spansMatch(h.NegativeSpans, h2.NegativeSpans) { return false } - if !floatBucketsMatch(h.NegativeBuckets, h2.NegativeBuckets) { + if !FloatBucketsMatch(h.NegativeBuckets, h2.NegativeBuckets) { return false } if !spansMatch(h.PositiveSpans, h2.PositiveSpans) { return false } - if !floatBucketsMatch(h.PositiveBuckets, h2.PositiveBuckets) { + if !FloatBucketsMatch(h.PositiveBuckets, h2.PositiveBuckets) { return false } @@ -490,7 +494,7 @@ func (h *FloatHistogram) Size() int { negSpanSize := len(h.NegativeSpans) * 8 // 8 bytes (int32 + uint32). posBucketSize := len(h.PositiveBuckets) * 8 // 8 bytes (float64). negBucketSize := len(h.NegativeBuckets) * 8 // 8 bytes (float64). - customBoundSize := len(h.CustomBounds) * 8 // 8 bytes (float64). + customBoundSize := len(h.CustomValues) * 8 // 8 bytes (float64). // Total size of the struct. @@ -505,7 +509,7 @@ func (h *FloatHistogram) Size() int { // fh.NegativeSpans is 24 bytes. // fh.PositiveBuckets is 24 bytes. // fh.NegativeBuckets is 24 bytes. - // fh.CustomBounds is 24 bytes. + // fh.CustomValues is 24 bytes. structSize := 168 return structSize + posSpanSize + negSpanSize + posBucketSize + negBucketSize + customBoundSize @@ -593,7 +597,7 @@ func (h *FloatHistogram) DetectReset(previous *FloatHistogram) bool { if h.Count < previous.Count { return true } - if h.UsesCustomBuckets() != previous.UsesCustomBuckets() || (h.UsesCustomBuckets() && !floatBucketsMatch(h.CustomBounds, previous.CustomBounds)) { + if h.UsesCustomBuckets() != previous.UsesCustomBuckets() || (h.UsesCustomBuckets() && !FloatBucketsMatch(h.CustomValues, previous.CustomValues)) { // Mark that something has changed or that the application has been restarted. However, this does // not matter so much since the change in schema will be handled directly in the chunks and PromQL // functions. @@ -704,7 +708,7 @@ func (h *FloatHistogram) NegativeBucketIterator() BucketIterator[float64] { // positive buckets in descending order (starting at the highest bucket and // going down towards the zero bucket). func (h *FloatHistogram) PositiveReverseBucketIterator() BucketIterator[float64] { - it := newReverseFloatBucketIterator(h.PositiveSpans, h.PositiveBuckets, h.Schema, true, h.CustomBounds) + it := newReverseFloatBucketIterator(h.PositiveSpans, h.PositiveBuckets, h.Schema, true, h.CustomValues) return &it } @@ -738,7 +742,7 @@ func (h *FloatHistogram) AllBucketIterator() BucketIterator[float64] { func (h *FloatHistogram) AllReverseBucketIterator() BucketIterator[float64] { return &allFloatBucketIterator{ h: h, - leftIter: newReverseFloatBucketIterator(h.PositiveSpans, h.PositiveBuckets, h.Schema, true, h.CustomBounds), + leftIter: newReverseFloatBucketIterator(h.PositiveSpans, h.PositiveBuckets, h.Schema, true, h.CustomValues), rightIter: h.floatBucketIterator(false, 0, h.Schema), state: -1, } @@ -753,7 +757,7 @@ func (h *FloatHistogram) AllReverseBucketIterator() BucketIterator[float64] { func (h *FloatHistogram) Validate() error { var nCount, pCount float64 if h.UsesCustomBuckets() { - if err := checkHistogramCustomBounds(h.CustomBounds, h.PositiveSpans, len(h.PositiveBuckets)); err != nil { + if err := checkHistogramCustomBounds(h.CustomValues, h.PositiveSpans, len(h.PositiveBuckets)); err != nil { return fmt.Errorf("custom buckets: %w", err) } if h.ZeroCount != 0 { @@ -779,7 +783,7 @@ func (h *FloatHistogram) Validate() error { if err != nil { return fmt.Errorf("negative side: %w", err) } - if h.CustomBounds != nil { + if h.CustomValues != nil { return fmt.Errorf("histogram with exponential schema must not have custom bounds") } } @@ -940,7 +944,7 @@ func (h *FloatHistogram) floatBucketIterator( if positive { i.spans = h.PositiveSpans i.buckets = h.PositiveBuckets - i.customBounds = h.CustomBounds + i.customValues = h.CustomValues } else { i.spans = h.NegativeSpans i.buckets = h.NegativeBuckets @@ -950,7 +954,7 @@ func (h *FloatHistogram) floatBucketIterator( // reverseFloatBucketIterator is a low-level constructor for reverse bucket iterators. func newReverseFloatBucketIterator( - spans []Span, buckets []float64, schema int32, positive bool, customBounds []float64, + spans []Span, buckets []float64, schema int32, positive bool, customValues []float64, ) reverseFloatBucketIterator { r := reverseFloatBucketIterator{ baseBucketIterator: baseBucketIterator[float64, float64]{ @@ -958,7 +962,7 @@ func newReverseFloatBucketIterator( spans: spans, buckets: buckets, positive: positive, - customBounds: customBounds, + customValues: customValues, }, } @@ -1296,7 +1300,7 @@ func addBuckets( return spansA, bucketsA } -func floatBucketsMatch(b1, b2 []float64) bool { +func FloatBucketsMatch(b1, b2 []float64) bool { if len(b1) != len(b2) { return false } diff --git a/model/histogram/float_histogram_test.go b/model/histogram/float_histogram_test.go index 400645cad4..3b69defe0f 100644 --- a/model/histogram/float_histogram_test.go +++ b/model/histogram/float_histogram_test.go @@ -139,7 +139,7 @@ func TestFloatHistogramMul(t *testing.T) { Sum: 2349209.324, PositiveSpans: []Span{{0, 1}, {2, 3}}, PositiveBuckets: []float64{1, 3.3, 4.2, 0.1}, - CustomBounds: []float64{1, 2, 3}, + CustomValues: []float64{1, 2, 3}, }, 1, &FloatHistogram{ @@ -148,7 +148,7 @@ func TestFloatHistogramMul(t *testing.T) { Sum: 2349209.324, PositiveSpans: []Span{{0, 1}, {2, 3}}, PositiveBuckets: []float64{1, 3.3, 4.2, 0.1}, - CustomBounds: []float64{1, 2, 3}, + CustomValues: []float64{1, 2, 3}, }, }, { @@ -159,7 +159,7 @@ func TestFloatHistogramMul(t *testing.T) { Sum: 23, PositiveSpans: []Span{{0, 2}, {1, 3}}, PositiveBuckets: []float64{1, 0, 3, 4, 7}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, 3, &FloatHistogram{ @@ -168,7 +168,7 @@ func TestFloatHistogramMul(t *testing.T) { Sum: 69, PositiveSpans: []Span{{0, 2}, {1, 3}}, PositiveBuckets: []float64{3, 0, 9, 12, 21}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, }, } @@ -224,13 +224,13 @@ func TestFloatHistogramCopy(t *testing.T) { Schema: CustomBucketsSchema, PositiveSpans: []Span{{0, 1}}, PositiveBuckets: []float64{1, 3, -3, 42}, - CustomBounds: []float64{1, 2, 3}, + CustomValues: []float64{1, 2, 3}, }, expected: &FloatHistogram{ Schema: CustomBucketsSchema, PositiveSpans: []Span{{0, 1}}, PositiveBuckets: []float64{1, 3, -3, 42}, - CustomBounds: []float64{1, 2, 3}, + CustomValues: []float64{1, 2, 3}, }, }, } @@ -289,13 +289,13 @@ func TestFloatHistogramCopyTo(t *testing.T) { Schema: CustomBucketsSchema, PositiveSpans: []Span{{0, 1}}, PositiveBuckets: []float64{1, 3, -3, 42}, - CustomBounds: []float64{1, 2, 3}, + CustomValues: []float64{1, 2, 3}, }, expected: &FloatHistogram{ Schema: CustomBucketsSchema, PositiveSpans: []Span{{0, 1}}, PositiveBuckets: []float64{1, 3, -3, 42}, - CustomBounds: []float64{1, 2, 3}, + CustomValues: []float64{1, 2, 3}, }, }, } @@ -417,7 +417,7 @@ func TestFloatHistogramDiv(t *testing.T) { Sum: 2349209.324, PositiveSpans: []Span{{0, 1}, {2, 3}}, PositiveBuckets: []float64{1, 3.3, 4.2, 0.1}, - CustomBounds: []float64{1, 2, 3}, + CustomValues: []float64{1, 2, 3}, }, 1, &FloatHistogram{ @@ -426,7 +426,7 @@ func TestFloatHistogramDiv(t *testing.T) { Sum: 2349209.324, PositiveSpans: []Span{{0, 1}, {2, 3}}, PositiveBuckets: []float64{1, 3.3, 4.2, 0.1}, - CustomBounds: []float64{1, 2, 3}, + CustomValues: []float64{1, 2, 3}, }, }, { @@ -437,7 +437,7 @@ func TestFloatHistogramDiv(t *testing.T) { Sum: 23, PositiveSpans: []Span{{0, 2}, {1, 3}}, PositiveBuckets: []float64{1, 0, 3, 4, 7}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, 2, &FloatHistogram{ @@ -446,7 +446,7 @@ func TestFloatHistogramDiv(t *testing.T) { Sum: 11.5, PositiveSpans: []Span{{0, 2}, {1, 3}}, PositiveBuckets: []float64{0.5, 0, 1.5, 2, 3.5}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, }, } @@ -1051,7 +1051,7 @@ func TestFloatHistogramDetectReset(t *testing.T) { "no buckets to some buckets with custom bounds", &FloatHistogram{ Schema: CustomBucketsSchema, - CustomBounds: []float64{1, 2, 3}, + CustomValues: []float64{1, 2, 3}, }, &FloatHistogram{ Schema: CustomBucketsSchema, @@ -1059,7 +1059,7 @@ func TestFloatHistogramDetectReset(t *testing.T) { Sum: 2349209.324, PositiveSpans: []Span{{0, 1}, {2, 3}}, PositiveBuckets: []float64{1, 3.3, 4.2, 0.1}, - CustomBounds: []float64{1, 2, 3}, + CustomValues: []float64{1, 2, 3}, }, false, }, @@ -1071,11 +1071,11 @@ func TestFloatHistogramDetectReset(t *testing.T) { Sum: 2349209.324, PositiveSpans: []Span{{0, 1}, {2, 3}}, PositiveBuckets: []float64{1, 3.3, 4.2, 0.1}, - CustomBounds: []float64{1, 2, 3}, + CustomValues: []float64{1, 2, 3}, }, &FloatHistogram{ Schema: CustomBucketsSchema, - CustomBounds: []float64{1, 2, 3}, + CustomValues: []float64{1, 2, 3}, }, true, }, @@ -1087,7 +1087,7 @@ func TestFloatHistogramDetectReset(t *testing.T) { Sum: 2349209.324, PositiveSpans: []Span{{0, 1}, {2, 3}}, PositiveBuckets: []float64{1, 3.3, 4.2, 0.1}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, &FloatHistogram{ Schema: CustomBucketsSchema, @@ -1095,7 +1095,7 @@ func TestFloatHistogramDetectReset(t *testing.T) { Sum: 2349209.324, PositiveSpans: []Span{{0, 2}, {1, 3}}, PositiveBuckets: []float64{1, 1.23, 3.3, 4.2, 0.1}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, false, }, @@ -1107,7 +1107,7 @@ func TestFloatHistogramDetectReset(t *testing.T) { Sum: 2349209.324, PositiveSpans: []Span{{0, 2}, {1, 3}}, PositiveBuckets: []float64{1, 1.23, 3.3, 4.2, 0.1}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, &FloatHistogram{ Schema: CustomBucketsSchema, @@ -1115,7 +1115,7 @@ func TestFloatHistogramDetectReset(t *testing.T) { Sum: 2349209.324, PositiveSpans: []Span{{0, 1}, {2, 3}}, PositiveBuckets: []float64{1, 3.3, 4.2, 0.1}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, true, }, @@ -1127,7 +1127,7 @@ func TestFloatHistogramDetectReset(t *testing.T) { Sum: 2349209.324, PositiveSpans: []Span{{0, 2}, {1, 3}}, PositiveBuckets: []float64{1, 0, 3.3, 4.2, 0.1}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, &FloatHistogram{ Schema: CustomBucketsSchema, @@ -1135,7 +1135,7 @@ func TestFloatHistogramDetectReset(t *testing.T) { Sum: 2349209.324, PositiveSpans: []Span{{0, 1}, {2, 3}}, PositiveBuckets: []float64{1, 3.3, 4.2, 0.1}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, false, }, @@ -1147,7 +1147,7 @@ func TestFloatHistogramDetectReset(t *testing.T) { Sum: 2349209.324, PositiveSpans: []Span{{0, 2}, {1, 3}}, PositiveBuckets: []float64{1, 0, 3.3, 4.2, 0.1}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, &FloatHistogram{ Schema: CustomBucketsSchema, @@ -1155,7 +1155,7 @@ func TestFloatHistogramDetectReset(t *testing.T) { Sum: 2349209.324, PositiveSpans: []Span{{0, 2}, {1, 3}}, PositiveBuckets: []float64{1, 0, 3.3, 4.3, 0.1}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, false, }, @@ -1167,7 +1167,7 @@ func TestFloatHistogramDetectReset(t *testing.T) { Sum: 2349209.324, PositiveSpans: []Span{{0, 2}, {1, 3}}, PositiveBuckets: []float64{1, 0, 3.3, 4.2, 0.1}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, &FloatHistogram{ Schema: CustomBucketsSchema, @@ -1175,7 +1175,7 @@ func TestFloatHistogramDetectReset(t *testing.T) { Sum: 2349209.324, PositiveSpans: []Span{{0, 2}, {1, 3}}, PositiveBuckets: []float64{1, 0, 3.3, 4.1, 0.1}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, true, }, @@ -1478,14 +1478,14 @@ func TestFloatHistogramCompact(t *testing.T) { Schema: CustomBucketsSchema, PositiveSpans: []Span{{0, 1}, {2, 3}}, PositiveBuckets: []float64{1, 3.3, 4.2, 0.1}, - CustomBounds: []float64{1, 2, 3}, + CustomValues: []float64{1, 2, 3}, }, 0, &FloatHistogram{ Schema: CustomBucketsSchema, PositiveSpans: []Span{{0, 1}, {2, 3}}, PositiveBuckets: []float64{1, 3.3, 4.2, 0.1}, - CustomBounds: []float64{1, 2, 3}, + CustomValues: []float64{1, 2, 3}, }, }, { @@ -1494,14 +1494,14 @@ func TestFloatHistogramCompact(t *testing.T) { Schema: CustomBucketsSchema, PositiveSpans: []Span{{0, 1}, {0, 3}, {0, 1}}, PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, 0, &FloatHistogram{ Schema: CustomBucketsSchema, PositiveSpans: []Span{{0, 5}}, PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, }, { @@ -1510,14 +1510,14 @@ func TestFloatHistogramCompact(t *testing.T) { Schema: CustomBucketsSchema, PositiveSpans: []Span{{0, 2}, {2, 0}, {2, 0}, {2, 0}, {3, 3}}, PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, 0, &FloatHistogram{ Schema: CustomBucketsSchema, PositiveSpans: []Span{{0, 2}, {9, 3}}, PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, }, { @@ -1526,14 +1526,14 @@ func TestFloatHistogramCompact(t *testing.T) { Schema: CustomBucketsSchema, PositiveSpans: []Span{{0, 4}, {5, 6}}, PositiveBuckets: []float64{0, 0, 1, 3.3, 4.2, 0.1, 3.3, 0, 0, 0}, - CustomBounds: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9}, + CustomValues: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9}, }, 0, &FloatHistogram{ Schema: CustomBucketsSchema, PositiveSpans: []Span{{2, 2}, {5, 3}}, PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3}, - CustomBounds: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9}, + CustomValues: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9}, }, }, } @@ -2009,7 +2009,7 @@ func TestFloatHistogramAdd(t *testing.T) { Sum: 2.345, PositiveSpans: []Span{{0, 2}, {1, 3}}, PositiveBuckets: []float64{1, 0, 3, 4, 7}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, &FloatHistogram{ Schema: CustomBucketsSchema, @@ -2017,7 +2017,7 @@ func TestFloatHistogramAdd(t *testing.T) { Sum: 1.234, PositiveSpans: []Span{{0, 2}, {1, 3}}, PositiveBuckets: []float64{0, 0, 2, 3, 6}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, &FloatHistogram{ Schema: CustomBucketsSchema, @@ -2025,7 +2025,7 @@ func TestFloatHistogramAdd(t *testing.T) { Sum: 3.579, PositiveSpans: []Span{{0, 2}, {1, 3}}, PositiveBuckets: []float64{1, 0, 5, 7, 13}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, "", }, @@ -2037,7 +2037,7 @@ func TestFloatHistogramAdd(t *testing.T) { Sum: 2.345, PositiveSpans: []Span{{0, 2}, {1, 1}, {0, 2}}, PositiveBuckets: []float64{1, 0, 3, 4, 7}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, &FloatHistogram{ Schema: CustomBucketsSchema, @@ -2045,7 +2045,7 @@ func TestFloatHistogramAdd(t *testing.T) { Sum: 1.234, PositiveSpans: []Span{{0, 2}, {1, 2}, {0, 1}}, PositiveBuckets: []float64{0, 0, 2, 3, 6}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, &FloatHistogram{ Schema: CustomBucketsSchema, @@ -2053,7 +2053,7 @@ func TestFloatHistogramAdd(t *testing.T) { Sum: 3.579, PositiveSpans: []Span{{0, 2}, {1, 1}, {0, 2}}, PositiveBuckets: []float64{1, 0, 5, 7, 13}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, "", }, @@ -2065,7 +2065,7 @@ func TestFloatHistogramAdd(t *testing.T) { Sum: 2.345, PositiveSpans: []Span{{0, 2}, {2, 3}}, PositiveBuckets: []float64{1, 0, 3, 4, 7}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, &FloatHistogram{ Schema: CustomBucketsSchema, @@ -2073,7 +2073,7 @@ func TestFloatHistogramAdd(t *testing.T) { Sum: 1.234, PositiveSpans: []Span{{2, 2}, {3, 3}}, PositiveBuckets: []float64{5, 4, 2, 3, 6}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, &FloatHistogram{ Schema: CustomBucketsSchema, @@ -2081,7 +2081,7 @@ func TestFloatHistogramAdd(t *testing.T) { Sum: 3.579, PositiveSpans: []Span{{0, 4}, {0, 6}}, PositiveBuckets: []float64{1, 0, 5, 4, 3, 4, 7, 2, 3, 6}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, "", }, @@ -2093,7 +2093,7 @@ func TestFloatHistogramAdd(t *testing.T) { Sum: 1.234, PositiveSpans: []Span{{2, 2}, {3, 3}}, PositiveBuckets: []float64{5, 4, 2, 3, 6}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, &FloatHistogram{ Schema: CustomBucketsSchema, @@ -2101,7 +2101,7 @@ func TestFloatHistogramAdd(t *testing.T) { Sum: 2.345, PositiveSpans: []Span{{0, 2}, {2, 3}}, PositiveBuckets: []float64{1, 0, 3, 4, 7}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, &FloatHistogram{ Schema: CustomBucketsSchema, @@ -2109,7 +2109,7 @@ func TestFloatHistogramAdd(t *testing.T) { Sum: 3.579, PositiveSpans: []Span{{0, 4}, {0, 6}}, PositiveBuckets: []float64{1, 0, 5, 4, 3, 4, 7, 2, 3, 6}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, "", }, @@ -2121,7 +2121,7 @@ func TestFloatHistogramAdd(t *testing.T) { Sum: 2.345, PositiveSpans: []Span{{0, 2}, {2, 3}}, PositiveBuckets: []float64{1, 0, 3, 4, 7}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, &FloatHistogram{ Schema: CustomBucketsSchema, @@ -2129,7 +2129,7 @@ func TestFloatHistogramAdd(t *testing.T) { Sum: 1.234, PositiveSpans: []Span{{1, 4}, {0, 3}}, PositiveBuckets: []float64{5, 4, 2, 3, 6, 2, 5}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, &FloatHistogram{ Schema: CustomBucketsSchema, @@ -2137,7 +2137,7 @@ func TestFloatHistogramAdd(t *testing.T) { Sum: 3.579, PositiveSpans: []Span{{0, 4}, {0, 4}}, PositiveBuckets: []float64{1, 5, 4, 2, 6, 10, 9, 5}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, "", }, @@ -2149,7 +2149,7 @@ func TestFloatHistogramAdd(t *testing.T) { Sum: 1.234, PositiveSpans: []Span{{1, 4}, {0, 3}}, PositiveBuckets: []float64{5, 4, 2, 3, 6, 2, 5}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, &FloatHistogram{ Schema: CustomBucketsSchema, @@ -2157,7 +2157,7 @@ func TestFloatHistogramAdd(t *testing.T) { Sum: 2.345, PositiveSpans: []Span{{0, 2}, {2, 3}}, PositiveBuckets: []float64{1, 0, 3, 4, 7}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, &FloatHistogram{ Schema: CustomBucketsSchema, @@ -2165,7 +2165,7 @@ func TestFloatHistogramAdd(t *testing.T) { Sum: 3.579, PositiveSpans: []Span{{0, 4}, {0, 4}}, PositiveBuckets: []float64{1, 5, 4, 2, 6, 10, 9, 5}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, "", }, @@ -2177,7 +2177,7 @@ func TestFloatHistogramAdd(t *testing.T) { Sum: 2.345, PositiveSpans: []Span{{0, 2}, {1, 3}}, PositiveBuckets: []float64{1, 0, 3, 4, 7}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, &FloatHistogram{ Schema: CustomBucketsSchema, @@ -2185,7 +2185,7 @@ func TestFloatHistogramAdd(t *testing.T) { Sum: 1.234, PositiveSpans: []Span{{0, 2}, {1, 3}}, PositiveBuckets: []float64{0, 0, 2, 3, 6}, - CustomBounds: []float64{1, 2, 3, 4, 5}, + CustomValues: []float64{1, 2, 3, 4, 5}, }, nil, "cannot apply this operation on custom buckets histograms with different custom bounds", @@ -2209,7 +2209,7 @@ func TestFloatHistogramAdd(t *testing.T) { Sum: 12, PositiveSpans: []Span{{0, 2}, {1, 3}}, PositiveBuckets: []float64{0, 0, 2, 3, 6}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, nil, "cannot apply this operation on histograms with a mix of exponential and custom bucket schemas", @@ -2342,7 +2342,7 @@ func TestFloatHistogramSub(t *testing.T) { Sum: 23, PositiveSpans: []Span{{0, 2}, {1, 3}}, PositiveBuckets: []float64{1, 0, 3, 4, 7}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, &FloatHistogram{ Schema: CustomBucketsSchema, @@ -2350,7 +2350,7 @@ func TestFloatHistogramSub(t *testing.T) { Sum: 12, PositiveSpans: []Span{{0, 2}, {1, 3}}, PositiveBuckets: []float64{0, 0, 2, 3, 6}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, &FloatHistogram{ Schema: CustomBucketsSchema, @@ -2358,7 +2358,7 @@ func TestFloatHistogramSub(t *testing.T) { Sum: 11, PositiveSpans: []Span{{0, 2}, {1, 3}}, PositiveBuckets: []float64{1, 0, 1, 1, 1}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, "", }, @@ -2370,7 +2370,7 @@ func TestFloatHistogramSub(t *testing.T) { Sum: 23, PositiveSpans: []Span{{0, 2}, {1, 3}}, PositiveBuckets: []float64{1, 0, 3, 4, 7}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, &FloatHistogram{ Schema: CustomBucketsSchema, @@ -2378,7 +2378,7 @@ func TestFloatHistogramSub(t *testing.T) { Sum: 12, PositiveSpans: []Span{{0, 2}, {1, 3}}, PositiveBuckets: []float64{0, 0, 2, 3, 6}, - CustomBounds: []float64{1, 2, 3, 4, 5}, + CustomValues: []float64{1, 2, 3, 4, 5}, }, nil, "cannot apply this operation on custom buckets histograms with different custom bounds", @@ -2402,7 +2402,7 @@ func TestFloatHistogramSub(t *testing.T) { Sum: 12, PositiveSpans: []Span{{0, 2}, {1, 3}}, PositiveBuckets: []float64{0, 0, 2, 3, 6}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, nil, "cannot apply this operation on histograms with a mix of exponential and custom bucket schemas", @@ -2521,7 +2521,7 @@ func TestFloatHistogramCopyToSchema(t *testing.T) { Schema: CustomBucketsSchema, PositiveSpans: []Span{{0, 3}, {5, 5}}, PositiveBuckets: []float64{1, 0, 0, 3, 2, 2, 3, 4}, - CustomBounds: []float64{1, 2, 3, 4, 5, 6, 7}, + CustomValues: []float64{1, 2, 3, 4, 5, 6, 7}, }, &FloatHistogram{ Count: 30, @@ -2529,7 +2529,7 @@ func TestFloatHistogramCopyToSchema(t *testing.T) { Schema: CustomBucketsSchema, PositiveSpans: []Span{{0, 3}, {5, 5}}, PositiveBuckets: []float64{1, 0, 0, 3, 2, 2, 3, 4}, - CustomBounds: []float64{1, 2, 3, 4, 5, 6, 7}, + CustomValues: []float64{1, 2, 3, 4, 5, 6, 7}, }, }, } @@ -3120,7 +3120,7 @@ func TestFloatCustomBucketsIterators(t *testing.T) { {Offset: 1, Length: 1}, }, PositiveBuckets: []float64{100, 344, 123, 55}, - CustomBounds: []float64{10, 25, 50, 100, 500}, + CustomValues: []float64{10, 25, 50, 100, 500}, }, expPositiveBuckets: []Bucket[float64]{ {Lower: math.Inf(-1), Upper: 10, LowerInclusive: true, UpperInclusive: true, Count: 100, Index: 0}, @@ -3140,7 +3140,7 @@ func TestFloatCustomBucketsIterators(t *testing.T) { {Offset: 1, Length: 1}, }, PositiveBuckets: []float64{100, 344, 123, 55}, - CustomBounds: []float64{-10, -5, 0, 10, 25}, + CustomValues: []float64{-10, -5, 0, 10, 25}, }, expPositiveBuckets: []Bucket[float64]{ {Lower: math.Inf(-1), Upper: -10, LowerInclusive: true, UpperInclusive: true, Count: 100, Index: 0}, @@ -3259,7 +3259,7 @@ func TestFloatHistogramEquals(t *testing.T) { // Custom bounds are defined for exponential schema. hCustom := h1.Copy() - hCustom.CustomBounds = []float64{1, 2, 3} + hCustom.CustomValues = []float64{1, 2, 3} equals(h1, *hCustom) cbh1 := FloatHistogram{ @@ -3268,7 +3268,7 @@ func TestFloatHistogramEquals(t *testing.T) { Sum: 9.7, PositiveSpans: []Span{{0, 1}}, PositiveBuckets: []float64{3}, - CustomBounds: []float64{1, 2, 3}, + CustomValues: []float64{1, 2, 3}, } require.NoError(t, cbh1.Validate()) @@ -3278,7 +3278,7 @@ func TestFloatHistogramEquals(t *testing.T) { // Has different custom bounds for custom buckets schema. cbh2 = cbh1.Copy() - cbh2.CustomBounds = []float64{1, 2, 3, 4} + cbh2.CustomValues = []float64{1, 2, 3, 4} notEquals(cbh1, *cbh2) // Has non-empty negative spans and buckets for custom buckets schema. @@ -3313,7 +3313,7 @@ func TestFloatHistogramSize(t *testing.T) { PositiveBuckets: nil, // 24 bytes. NegativeSpans: nil, // 24 bytes. NegativeBuckets: nil, // 24 bytes. - CustomBounds: nil, // 24 bytes. + CustomValues: nil, // 24 bytes. }, 8 + 4 + 4 + 8 + 8 + 8 + 8 + 24 + 24 + 24 + 24 + 24, }, @@ -3335,7 +3335,7 @@ func TestFloatHistogramSize(t *testing.T) { {3, 2}, // 2 * 4 bytes. {3, 2}}, // 2 * 4 bytes. NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000}, // 24 bytes + 4 * 8 bytes. - CustomBounds: nil, // 24 bytes. + CustomValues: nil, // 24 bytes. }, 8 + 4 + 4 + 8 + 8 + 8 + 8 + (24 + 2*4 + 2*4) + (24 + 2*4 + 2*4) + (24 + 4*8) + (24 + 4*8) + 24, }, @@ -3355,7 +3355,7 @@ func TestFloatHistogramSize(t *testing.T) { PositiveBuckets: []float64{1, 3.3, 4.2, 0.1}, // 24 bytes + 4 * 8 bytes. NegativeSpans: nil, // 24 bytes. NegativeBuckets: nil, // 24 bytes. - CustomBounds: []float64{1, 2, 3}, // 24 bytes + 3 * 8 bytes. + CustomValues: []float64{1, 2, 3}, // 24 bytes + 3 * 8 bytes. }, 8 + 4 + 4 + 8 + 8 + 8 + 8 + (24 + 2*4 + 2*4) + (24 + 4*8) + 24 + 24 + (24 + 3*8), }, @@ -3406,7 +3406,7 @@ func TestFloatHistogramString(t *testing.T) { {2, 4}, }, PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 5}, - CustomBounds: []float64{1, 2, 5, 10, 15, 20}, + CustomValues: []float64{1, 2, 5, 10, 15, 20}, }, `{count:3493.3, sum:2.349209324e+06, [-Inf,1]:1, (5,10]:3.3, (10,15]:4.2, (15,20]:0.1, (20,+Inf]:5}`, }, diff --git a/model/histogram/generic.go b/model/histogram/generic.go index c6780c2003..025888ccae 100644 --- a/model/histogram/generic.go +++ b/model/histogram/generic.go @@ -23,7 +23,7 @@ import ( const ( ExponentialSchemaMax int32 = 8 ExponentialSchemaMin int32 = -4 - CustomBucketsSchema int32 = 127 + CustomBucketsSchema int32 = -53 ) var ( @@ -134,7 +134,7 @@ type baseBucketIterator[BC BucketCount, IBC InternalBucketCount] struct { currCount IBC // Count in the current bucket. currIdx int32 // The actual bucket index. - customBounds []float64 // Bounds (usually upper) for histograms with custom buckets. + customValues []float64 // Bounds (usually upper) for histograms with custom buckets. } func (b *baseBucketIterator[BC, IBC]) At() Bucket[BC] { @@ -148,11 +148,11 @@ func (b *baseBucketIterator[BC, IBC]) at(schema int32) Bucket[BC] { Index: b.currIdx, } if b.positive { - bucket.Upper = getBound(b.currIdx, schema, b.customBounds) - bucket.Lower = getBound(b.currIdx-1, schema, b.customBounds) + bucket.Upper = getBound(b.currIdx, schema, b.customValues) + bucket.Lower = getBound(b.currIdx-1, schema, b.customValues) } else { - bucket.Lower = -getBound(b.currIdx, schema, b.customBounds) - bucket.Upper = -getBound(b.currIdx-1, schema, b.customBounds) + bucket.Lower = -getBound(b.currIdx, schema, b.customValues) + bucket.Upper = -getBound(b.currIdx-1, schema, b.customValues) } if IsCustomBucketsSchema(schema) { bucket.LowerInclusive = b.currIdx == 0 @@ -446,9 +446,9 @@ func checkHistogramCustomBounds(bounds []float64, spans []Span, numBuckets int) return nil } -func getBound(idx, schema int32, customBounds []float64) float64 { +func getBound(idx, schema int32, customValues []float64) float64 { if IsCustomBucketsSchema(schema) { - length := int32(len(customBounds)) + length := int32(len(customValues)) switch { case idx > length || idx < -1: panic(fmt.Errorf("index %d out of bounds for custom bounds of length %d", idx, length)) @@ -457,7 +457,7 @@ func getBound(idx, schema int32, customBounds []float64) float64 { case idx == -1: return math.Inf(-1) default: - return customBounds[idx] + return customValues[idx] } } return getBoundExponential(idx, schema) diff --git a/model/histogram/histogram.go b/model/histogram/histogram.go index 98a8a606c9..17cd036c04 100644 --- a/model/histogram/histogram.go +++ b/model/histogram/histogram.go @@ -54,8 +54,8 @@ type Histogram struct { // They are all for base-2 bucket schemas, where 1 is a bucket boundary in // each case, and then each power of two is divided into 2^n logarithmic buckets. // Or in other words, each bucket boundary is the previous boundary times - // 2^(2^-n). Another valid schema number is 127 for custom buckets, defined by - // the CustomBounds field. + // 2^(2^-n). Another valid schema number is -53 for custom buckets, defined by + // the CustomValues field. Schema int32 // Width of the zero bucket. ZeroThreshold float64 @@ -74,9 +74,9 @@ type Histogram struct { // Holds the custom (usually upper) bounds for bucket definitions, otherwise nil. // This slice is interned, to be treated as immutable and copied by reference. // These numbers should be strictly increasing. This field is only used when the - // schema is 127, and the ZeroThreshold, ZeroCount, NegativeSpans and NegativeBuckets - // fields are not used. - CustomBounds []float64 + // schema is for custom buckets, and the ZeroThreshold, ZeroCount, NegativeSpans + // and NegativeBuckets fields are not used. + CustomValues []float64 } // A Span defines a continuous sequence of buckets. @@ -102,7 +102,10 @@ func (h *Histogram) Copy() *Histogram { } if h.UsesCustomBuckets() { - c.CustomBounds = h.CustomBounds + if len(h.CustomValues) != 0 { + c.CustomValues = make([]float64, len(h.CustomValues)) + copy(c.CustomValues, h.CustomValues) + } } else { c.ZeroThreshold = h.ZeroThreshold c.ZeroCount = h.ZeroCount @@ -144,7 +147,8 @@ func (h *Histogram) CopyTo(to *Histogram) { to.NegativeSpans = clearIfNotNil(to.NegativeSpans) to.NegativeBuckets = clearIfNotNil(to.NegativeBuckets) - to.CustomBounds = h.CustomBounds + to.CustomValues = resize(to.CustomValues, len(h.CustomValues)) + copy(to.CustomValues, h.CustomValues) } else { to.ZeroThreshold = h.ZeroThreshold to.ZeroCount = h.ZeroCount @@ -155,7 +159,7 @@ func (h *Histogram) CopyTo(to *Histogram) { to.NegativeBuckets = resize(to.NegativeBuckets, len(h.NegativeBuckets)) copy(to.NegativeBuckets, h.NegativeBuckets) - to.CustomBounds = clearIfNotNil(to.CustomBounds) + to.CustomValues = clearIfNotNil(to.CustomValues) } to.PositiveSpans = resize(to.PositiveSpans, len(h.PositiveSpans)) @@ -213,7 +217,7 @@ func (h *Histogram) ZeroBucket() Bucket[uint64] { // PositiveBucketIterator returns a BucketIterator to iterate over all positive // buckets in ascending order (starting next to the zero bucket and going up). func (h *Histogram) PositiveBucketIterator() BucketIterator[uint64] { - it := newRegularBucketIterator(h.PositiveSpans, h.PositiveBuckets, h.Schema, true, h.CustomBounds) + it := newRegularBucketIterator(h.PositiveSpans, h.PositiveBuckets, h.Schema, true, h.CustomValues) return &it } @@ -255,7 +259,7 @@ func (h *Histogram) Equals(h2 *Histogram) bool { } if h.UsesCustomBuckets() { - if !floatBucketsMatch(h.CustomBounds, h2.CustomBounds) { + if !FloatBucketsMatch(h.CustomValues, h2.CustomValues) { return false } } @@ -375,7 +379,9 @@ func (h *Histogram) ToFloat(fh *FloatHistogram) *FloatHistogram { fh.ZeroCount = 0 fh.NegativeSpans = clearIfNotNil(fh.NegativeSpans) fh.NegativeBuckets = clearIfNotNil(fh.NegativeBuckets) - fh.CustomBounds = h.CustomBounds + + fh.CustomValues = resize(fh.CustomValues, len(h.CustomValues)) + copy(fh.CustomValues, h.CustomValues) } else { fh.ZeroThreshold = h.ZeroThreshold fh.ZeroCount = float64(h.ZeroCount) @@ -389,7 +395,7 @@ func (h *Histogram) ToFloat(fh *FloatHistogram) *FloatHistogram { currentNegative += float64(b) fh.NegativeBuckets[i] = currentNegative } - fh.CustomBounds = clearIfNotNil(fh.CustomBounds) + fh.CustomValues = clearIfNotNil(fh.CustomValues) } fh.PositiveSpans = resize(fh.PositiveSpans, len(h.PositiveSpans)) @@ -430,7 +436,7 @@ func clearIfNotNil[T any](items []T) []T { func (h *Histogram) Validate() error { var nCount, pCount uint64 if h.UsesCustomBuckets() { - if err := checkHistogramCustomBounds(h.CustomBounds, h.PositiveSpans, len(h.PositiveBuckets)); err != nil { + if err := checkHistogramCustomBounds(h.CustomValues, h.PositiveSpans, len(h.PositiveBuckets)); err != nil { return fmt.Errorf("custom buckets: %w", err) } if h.ZeroCount != 0 { @@ -456,7 +462,7 @@ func (h *Histogram) Validate() error { if err != nil { return fmt.Errorf("negative side: %w", err) } - if h.CustomBounds != nil { + if h.CustomValues != nil { return fmt.Errorf("histogram with exponential schema must not have custom bounds") } } @@ -483,13 +489,13 @@ type regularBucketIterator struct { baseBucketIterator[uint64, int64] } -func newRegularBucketIterator(spans []Span, buckets []int64, schema int32, positive bool, customBounds []float64) regularBucketIterator { +func newRegularBucketIterator(spans []Span, buckets []int64, schema int32, positive bool, customValues []float64) regularBucketIterator { i := baseBucketIterator[uint64, int64]{ schema: schema, spans: spans, buckets: buckets, positive: positive, - customBounds: customBounds, + customValues: customValues, } return regularBucketIterator{i} } @@ -563,7 +569,7 @@ func (c *cumulativeBucketIterator) Next() bool { if c.emptyBucketCount > 0 { // We are traversing through empty buckets at the moment. - c.currUpper = getBound(c.currIdx, c.h.Schema, c.h.CustomBounds) + c.currUpper = getBound(c.currIdx, c.h.Schema, c.h.CustomValues) c.currIdx++ c.emptyBucketCount-- return true @@ -580,7 +586,7 @@ func (c *cumulativeBucketIterator) Next() bool { c.currCount += c.h.PositiveBuckets[c.posBucketsIdx] c.currCumulativeCount += uint64(c.currCount) - c.currUpper = getBound(c.currIdx, c.h.Schema, c.h.CustomBounds) + c.currUpper = getBound(c.currIdx, c.h.Schema, c.h.CustomValues) c.posBucketsIdx++ c.idxInSpan++ diff --git a/model/histogram/histogram_test.go b/model/histogram/histogram_test.go index e63819debc..2c276e6c92 100644 --- a/model/histogram/histogram_test.go +++ b/model/histogram/histogram_test.go @@ -80,7 +80,7 @@ func TestHistogramString(t *testing.T) { {Offset: 0, Length: 3}, }, PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 0}, - CustomBounds: []float64{1, 2, 5, 10, 15, 20, 25, 50}, + CustomValues: []float64{1, 2, 5, 10, 15, 20, 25, 50}, }, expectedString: "{count:19, sum:2.7, [-Inf,1]:1, (1,2]:3, (2,5]:1, (5,10]:2, (10,15]:1, (15,20]:1, (20,25]:1}", }, @@ -231,7 +231,7 @@ func TestCumulativeBucketIterator(t *testing.T) { {Offset: 1, Length: 2}, }, PositiveBuckets: []int64{1, 1, -1, 0}, - CustomBounds: []float64{5, 10, 20, 50}, + CustomValues: []float64{5, 10, 20, 50}, }, expectedBuckets: []Bucket[uint64]{ {Lower: math.Inf(-1), Upper: 5, Count: 1, LowerInclusive: true, UpperInclusive: true, Index: 0}, @@ -411,7 +411,7 @@ func TestRegularBucketIterator(t *testing.T) { {Offset: 1, Length: 2}, }, PositiveBuckets: []int64{1, 1, -1, 0}, - CustomBounds: []float64{5, 10, 20, 50}, + CustomValues: []float64{5, 10, 20, 50}, }, expectedPositiveBuckets: []Bucket[uint64]{ {Lower: math.Inf(-1), Upper: 5, Count: 1, LowerInclusive: true, UpperInclusive: true, Index: 0}, @@ -430,7 +430,7 @@ func TestRegularBucketIterator(t *testing.T) { {Offset: 1, Length: 2}, }, PositiveBuckets: []int64{1, 1, -1, 0}, - CustomBounds: []float64{0, 10, 20, 50}, + CustomValues: []float64{0, 10, 20, 50}, }, expectedPositiveBuckets: []Bucket[uint64]{ {Lower: math.Inf(-1), Upper: 0, Count: 1, LowerInclusive: true, UpperInclusive: true, Index: 0}, @@ -448,7 +448,7 @@ func TestRegularBucketIterator(t *testing.T) { {Offset: 0, Length: 5}, }, PositiveBuckets: []int64{1, 1, 0, -1, 0}, - CustomBounds: []float64{-5, 0, 20, 50}, + CustomValues: []float64{-5, 0, 20, 50}, }, expectedPositiveBuckets: []Bucket[uint64]{ {Lower: math.Inf(-1), Upper: -5, Count: 1, LowerInclusive: true, UpperInclusive: true, Index: 0}, @@ -563,7 +563,7 @@ func TestCustomBucketsHistogramToFloat(t *testing.T) { {Offset: 0, Length: 3}, }, PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 0}, - CustomBounds: []float64{5, 10, 20, 50, 100, 500}, + CustomValues: []float64{5, 10, 20, 50, 100, 500}, } cases := []struct { name string @@ -773,7 +773,7 @@ func TestHistogramEquals(t *testing.T) { // Has non-empty custom bounds for exponential schema. hCustom := h1.Copy() - hCustom.CustomBounds = []float64{1, 2, 3} + hCustom.CustomValues = []float64{1, 2, 3} equals(h1, *hCustom) cbh1 := Histogram{ @@ -785,7 +785,7 @@ func TestHistogramEquals(t *testing.T) { {Offset: 10, Length: 3}, }, PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 0}, - CustomBounds: []float64{0.1, 0.2, 0.5, 1, 2, 5, 10, 15, 20, 25, 50, 75, 100, 200, 250, 500, 1000}, + CustomValues: []float64{0.1, 0.2, 0.5, 1, 2, 5, 10, 15, 20, 25, 50, 75, 100, 200, 250, 500, 1000}, } require.NoError(t, cbh1.Validate()) @@ -795,7 +795,7 @@ func TestHistogramEquals(t *testing.T) { // Has different custom bounds for custom buckets schema. cbh2 = cbh1.Copy() - cbh2.CustomBounds = []float64{0.1, 0.2, 0.5} + cbh2.CustomValues = []float64{0.1, 0.2, 0.5} notEquals(cbh1, *cbh2) // Has non-empty negative spans and buckets for custom buckets schema. @@ -853,13 +853,13 @@ func TestHistogramCopy(t *testing.T) { Schema: CustomBucketsSchema, PositiveSpans: []Span{{-2, 1}}, PositiveBuckets: []int64{1, 3, -3, 42}, - CustomBounds: []float64{5, 10, 15}, + CustomValues: []float64{5, 10, 15}, }, expected: &Histogram{ Schema: CustomBucketsSchema, PositiveSpans: []Span{{-2, 1}}, PositiveBuckets: []int64{1, 3, -3, 42}, - CustomBounds: []float64{5, 10, 15}, + CustomValues: []float64{5, 10, 15}, }, }, } @@ -918,13 +918,13 @@ func TestHistogramCopyTo(t *testing.T) { Schema: CustomBucketsSchema, PositiveSpans: []Span{{-2, 1}}, PositiveBuckets: []int64{1, 3, -3, 42}, - CustomBounds: []float64{5, 10, 15}, + CustomValues: []float64{5, 10, 15}, }, expected: &Histogram{ Schema: CustomBucketsSchema, PositiveSpans: []Span{{-2, 1}}, PositiveBuckets: []int64{1, 3, -3, 42}, - CustomBounds: []float64{5, 10, 15}, + CustomValues: []float64{5, 10, 15}, }, }, } @@ -1214,14 +1214,14 @@ func TestHistogramCompact(t *testing.T) { Schema: CustomBucketsSchema, PositiveSpans: []Span{{-2, 1}, {2, 3}}, PositiveBuckets: []int64{1, 3, -3, 42}, - CustomBounds: []float64{5, 10, 15}, + CustomValues: []float64{5, 10, 15}, }, 0, &Histogram{ Schema: CustomBucketsSchema, PositiveSpans: []Span{{-2, 1}, {2, 3}}, PositiveBuckets: []int64{1, 3, -3, 42}, - CustomBounds: []float64{5, 10, 15}, + CustomValues: []float64{5, 10, 15}, }, }, { @@ -1230,14 +1230,14 @@ func TestHistogramCompact(t *testing.T) { Schema: CustomBucketsSchema, PositiveSpans: []Span{{-2, 1}, {0, 3}, {0, 1}}, PositiveBuckets: []int64{1, 3, -3, 42, 3}, - CustomBounds: []float64{5, 10, 15, 20}, + CustomValues: []float64{5, 10, 15, 20}, }, 0, &Histogram{ Schema: CustomBucketsSchema, PositiveSpans: []Span{{-2, 5}}, PositiveBuckets: []int64{1, 3, -3, 42, 3}, - CustomBounds: []float64{5, 10, 15, 20}, + CustomValues: []float64{5, 10, 15, 20}, }, }, { @@ -1246,14 +1246,14 @@ func TestHistogramCompact(t *testing.T) { Schema: CustomBucketsSchema, PositiveSpans: []Span{{-2, 2}, {2, 0}, {3, 3}}, PositiveBuckets: []int64{1, 3, -3, 42, 3}, - CustomBounds: []float64{5, 10, 15, 20}, + CustomValues: []float64{5, 10, 15, 20}, }, 0, &Histogram{ Schema: CustomBucketsSchema, PositiveSpans: []Span{{-2, 2}, {5, 3}}, PositiveBuckets: []int64{1, 3, -3, 42, 3}, - CustomBounds: []float64{5, 10, 15, 20}, + CustomValues: []float64{5, 10, 15, 20}, }, }, { @@ -1262,14 +1262,14 @@ func TestHistogramCompact(t *testing.T) { Schema: CustomBucketsSchema, PositiveSpans: []Span{{-2, 2}, {2, 0}, {2, 0}, {2, 0}, {3, 3}}, PositiveBuckets: []int64{1, 3, -3, 42, 3}, - CustomBounds: []float64{5, 10, 15, 20}, + CustomValues: []float64{5, 10, 15, 20}, }, 0, &Histogram{ Schema: CustomBucketsSchema, PositiveSpans: []Span{{-2, 2}, {9, 3}}, PositiveBuckets: []int64{1, 3, -3, 42, 3}, - CustomBounds: []float64{5, 10, 15, 20}, + CustomValues: []float64{5, 10, 15, 20}, }, }, { @@ -1278,14 +1278,14 @@ func TestHistogramCompact(t *testing.T) { Schema: CustomBucketsSchema, PositiveSpans: []Span{{-4, 6}, {3, 6}}, PositiveBuckets: []int64{0, 0, 1, 3, -4, 0, 1, 42, 3, -46, 0, 0}, - CustomBounds: []float64{5, 10, 15, 20}, + CustomValues: []float64{5, 10, 15, 20}, }, 0, &Histogram{ Schema: CustomBucketsSchema, PositiveSpans: []Span{{-2, 2}, {5, 3}}, PositiveBuckets: []int64{1, 3, -3, 42, 3}, - CustomBounds: []float64{5, 10, 15, 20}, + CustomValues: []float64{5, 10, 15, 20}, }, }, } @@ -1454,7 +1454,7 @@ func TestHistogramValidation(t *testing.T) { {Offset: 1, Length: 2}, }, PositiveBuckets: []int64{1, 1, -1, 0}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, errMsg: `histogram with exponential schema must not have custom bounds`, skipFloat: true, // Converting to float will remove the wrong fields so only the float version will pass validation @@ -1476,7 +1476,7 @@ func TestHistogramValidation(t *testing.T) { {Offset: 1, Length: 2}, }, NegativeBuckets: []int64{1, 1, -1, 0}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, errMsg: `custom buckets: must have zero count of 0`, skipFloat: true, // Converting to float will remove the wrong fields so only the float version will pass validation @@ -1491,7 +1491,7 @@ func TestHistogramValidation(t *testing.T) { {Offset: 1, Length: 2}, }, PositiveBuckets: []int64{1, 1, -1, 0}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, errMsg: `custom buckets: span number 1 with offset -1: histogram has a span whose offset is negative`, }, @@ -1505,7 +1505,7 @@ func TestHistogramValidation(t *testing.T) { {Offset: -1, Length: 2}, }, PositiveBuckets: []int64{1, 1, -1, 0}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, errMsg: `custom buckets: span number 2 with offset -1: histogram has a span whose offset is negative`, }, @@ -1519,7 +1519,7 @@ func TestHistogramValidation(t *testing.T) { {Offset: 1, Length: 2}, }, PositiveBuckets: []int64{1, 1, -1}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, errMsg: `custom buckets: spans need 4 buckets, have 3 buckets: histogram spans specify different number of buckets than provided`, }, @@ -1533,7 +1533,7 @@ func TestHistogramValidation(t *testing.T) { {Offset: 1, Length: 2}, }, PositiveBuckets: []int64{1, 1, -1, 0}, - CustomBounds: []float64{1, 2, 3}, + CustomValues: []float64{1, 2, 3}, }, errMsg: `custom buckets: only 3 custom bounds defined which is insufficient to cover total span length of 5: histogram custom bounds are too few`, }, @@ -1547,7 +1547,7 @@ func TestHistogramValidation(t *testing.T) { {Offset: 1, Length: 2}, }, PositiveBuckets: []int64{1, 1, -1, 0}, - CustomBounds: []float64{1, 2, 3, 4}, + CustomValues: []float64{1, 2, 3, 4}, }, }, "valid custom buckets histogram with extra bounds": { @@ -1560,7 +1560,7 @@ func TestHistogramValidation(t *testing.T) { {Offset: 1, Length: 2}, }, PositiveBuckets: []int64{1, 1, -1, 0}, - CustomBounds: []float64{1, 2, 3, 4, 5, 6, 7, 8}, + CustomValues: []float64{1, 2, 3, 4, 5, 6, 7, 8}, }, }, } diff --git a/scrape/target_test.go b/scrape/target_test.go index dead3c6fc4..3457af7f0d 100644 --- a/scrape/target_test.go +++ b/scrape/target_test.go @@ -481,7 +481,7 @@ func TestBucketLimitAppender(t *testing.T) { {Offset: 0, Length: 3}, }, PositiveBuckets: []int64{3, 0, 0}, - CustomBounds: []float64{1, 2, 3}, + CustomValues: []float64{1, 2, 3}, } cases := []struct { @@ -592,7 +592,7 @@ func TestMaxSchemaAppender(t *testing.T) { {Offset: 0, Length: 3}, }, PositiveBuckets: []int64{3, 0, 0}, - CustomBounds: []float64{1, 2, 3}, + CustomValues: []float64{1, 2, 3}, } cases := []struct { diff --git a/tsdb/chunkenc/bstream.go b/tsdb/chunkenc/bstream.go index 7b17f4686b..6e13c40464 100644 --- a/tsdb/chunkenc/bstream.go +++ b/tsdb/chunkenc/bstream.go @@ -113,6 +113,16 @@ func (b *bstream) writeBits(u uint64, nbits int) { } } +// wrapper for the standard library's PutUvarint to make it work +// with our bstream. +func (b *bstream) putUvarint(x uint64) { + buf := make([]byte, 2) + l := binary.PutUvarint(buf, x) + for i := 0; i < l; i++ { + b.writeByte(buf[i]) + } +} + type bstreamReader struct { stream []byte streamOffset int // The offset from which read the next byte from the stream. @@ -257,3 +267,9 @@ func (b *bstreamReader) loadNextBuffer(nbits uint8) bool { return true } + +// wrapper for the standard library's ReadUvarint to make it work +// with our bstream. +func (b *bstreamReader) readUvarint() (uint64, error) { + return binary.ReadUvarint(b) +} diff --git a/tsdb/chunkenc/float_histogram.go b/tsdb/chunkenc/float_histogram.go index 88d189254f..b5ac4ae6ed 100644 --- a/tsdb/chunkenc/float_histogram.go +++ b/tsdb/chunkenc/float_histogram.go @@ -72,6 +72,7 @@ func (c *FloatHistogramChunk) NumSamples() int { func (c *FloatHistogramChunk) Layout() ( schema int32, zeroThreshold float64, negativeSpans, positiveSpans []histogram.Span, + customValues []float64, err error, ) { if c.NumSamples() == 0 { @@ -129,17 +130,18 @@ func (c *FloatHistogramChunk) Appender() (Appender, error) { a := &FloatHistogramAppender{ b: &c.b, - schema: it.schema, - zThreshold: it.zThreshold, - pSpans: it.pSpans, - nSpans: it.nSpans, - t: it.t, - tDelta: it.tDelta, - cnt: it.cnt, - zCnt: it.zCnt, - pBuckets: pBuckets, - nBuckets: nBuckets, - sum: it.sum, + schema: it.schema, + zThreshold: it.zThreshold, + pSpans: it.pSpans, + nSpans: it.nSpans, + customValues: it.customValues, + t: it.t, + tDelta: it.tDelta, + cnt: it.cnt, + zCnt: it.zCnt, + pBuckets: pBuckets, + nBuckets: nBuckets, + sum: it.sum, } if it.numTotal == 0 { a.sum.leading = 0xff @@ -187,6 +189,7 @@ type FloatHistogramAppender struct { schema int32 zThreshold float64 pSpans, nSpans []histogram.Span + customValues []float64 t, tDelta int64 sum, cnt, zCnt xorValue @@ -218,6 +221,7 @@ func (a *FloatHistogramAppender) Append(int64, float64) { // // The chunk is not appendable in the following cases: // - The schema has changed. +// - The custom bounds have changed if the current schema is custom buckets. // - The threshold for the zero bucket has changed. // - Any buckets have disappeared. // - There was a counter reset in the count of observations or in any bucket, including the zero bucket. @@ -259,6 +263,11 @@ func (a *FloatHistogramAppender) appendable(h *histogram.FloatHistogram) ( return } + if histogram.IsCustomBucketsSchema(h.Schema) && !histogram.FloatBucketsMatch(h.CustomValues, a.customValues) { + counterReset = true + return + } + if h.ZeroCount < a.zCnt.value { // There has been a counter reset since ZeroThreshold didn't change. counterReset = true @@ -299,6 +308,7 @@ func (a *FloatHistogramAppender) appendable(h *histogram.FloatHistogram) ( // // The chunk is not appendable in the following cases: // - The schema has changed. +// - The custom bounds have changed if the current schema is custom buckets. // - The threshold for the zero bucket has changed. // - The last sample in the chunk was stale while the current sample is not stale. func (a *FloatHistogramAppender) appendableGauge(h *histogram.FloatHistogram) ( @@ -325,6 +335,10 @@ func (a *FloatHistogramAppender) appendableGauge(h *histogram.FloatHistogram) ( return } + if histogram.IsCustomBucketsSchema(h.Schema) && !histogram.FloatBucketsMatch(h.CustomValues, a.customValues) { + return + } + positiveInserts, backwardPositiveInserts, positiveSpans = expandSpansBothWays(a.pSpans, h.PositiveSpans) negativeInserts, backwardNegativeInserts, negativeSpans = expandSpansBothWays(a.nSpans, h.NegativeSpans) okToAppend = true @@ -418,7 +432,7 @@ func (a *FloatHistogramAppender) appendFloatHistogram(t int64, h *histogram.Floa if num == 0 { // The first append gets the privilege to dictate the layout // but it's also responsible for encoding it into the chunk! - writeHistogramChunkLayout(a.b, h.Schema, h.ZeroThreshold, h.PositiveSpans, h.NegativeSpans) + writeHistogramChunkLayout(a.b, h.Schema, h.ZeroThreshold, h.PositiveSpans, h.NegativeSpans, h.CustomValues) a.schema = h.Schema a.zThreshold = h.ZeroThreshold @@ -434,6 +448,12 @@ func (a *FloatHistogramAppender) appendFloatHistogram(t int64, h *histogram.Floa } else { a.nSpans = nil } + if len(h.CustomValues) > 0 { + a.customValues = make([]float64, len(h.CustomValues)) + copy(a.customValues, h.CustomValues) + } else { + a.customValues = nil + } numPBuckets, numNBuckets := countSpans(h.PositiveSpans), countSpans(h.NegativeSpans) if numPBuckets > 0 { @@ -689,6 +709,7 @@ type floatHistogramIterator struct { schema int32 zThreshold float64 pSpans, nSpans []histogram.Span + customValues []float64 // For the fields that are tracked as deltas and ultimately dod's. t int64 @@ -749,6 +770,7 @@ func (it *floatHistogramIterator) AtFloatHistogram(fh *histogram.FloatHistogram) NegativeSpans: it.nSpans, PositiveBuckets: it.pBuckets, NegativeBuckets: it.nBuckets, + CustomValues: it.customValues, } } @@ -771,6 +793,9 @@ func (it *floatHistogramIterator) AtFloatHistogram(fh *histogram.FloatHistogram) fh.NegativeBuckets = resize(fh.NegativeBuckets, len(it.nBuckets)) copy(fh.NegativeBuckets, it.nBuckets) + fh.CustomValues = resize(fh.CustomValues, len(it.customValues)) + copy(fh.CustomValues, it.customValues) + return it.t, fh } @@ -815,7 +840,7 @@ func (it *floatHistogramIterator) Next() ValueType { // The first read is responsible for reading the chunk layout // and for initializing fields that depend on it. We give // counter reset info at chunk level, hence we discard it here. - schema, zeroThreshold, posSpans, negSpans, err := readHistogramChunkLayout(&it.br) + schema, zeroThreshold, posSpans, negSpans, customValues, err := readHistogramChunkLayout(&it.br) if err != nil { it.err = err return ValNone @@ -823,6 +848,7 @@ func (it *floatHistogramIterator) Next() ValueType { it.schema = schema it.zThreshold = zeroThreshold it.pSpans, it.nSpans = posSpans, negSpans + it.customValues = customValues numPBuckets, numNBuckets := countSpans(posSpans), countSpans(negSpans) // Allocate bucket slices as needed, recycling existing slices // in case this iterator was reset and already has slices of a diff --git a/tsdb/chunkenc/float_histogram_test.go b/tsdb/chunkenc/float_histogram_test.go index 054c17f7d9..2ee4422b91 100644 --- a/tsdb/chunkenc/float_histogram_test.go +++ b/tsdb/chunkenc/float_histogram_test.go @@ -280,7 +280,38 @@ func TestFloatHistogramChunkBucketChanges(t *testing.T) { } func TestFloatHistogramChunkAppendable(t *testing.T) { - setup := func() (Chunk, *FloatHistogramAppender, int64, *histogram.FloatHistogram) { + eh := &histogram.FloatHistogram{ + Count: 5, + ZeroCount: 2, + Sum: 18.4, + ZeroThreshold: 1e-125, + Schema: 1, + PositiveSpans: []histogram.Span{ + {Offset: 0, Length: 2}, + {Offset: 2, Length: 1}, + {Offset: 3, Length: 2}, + {Offset: 3, Length: 1}, + {Offset: 1, Length: 1}, + }, + PositiveBuckets: []float64{6, 3, 3, 2, 4, 5, 1}, + } + + cbh := &histogram.FloatHistogram{ + Count: 24, + Sum: 18.4, + Schema: histogram.CustomBucketsSchema, + PositiveSpans: []histogram.Span{ + {Offset: 0, Length: 2}, + {Offset: 2, Length: 1}, + {Offset: 3, Length: 2}, + {Offset: 3, Length: 1}, + {Offset: 1, Length: 1}, + }, + PositiveBuckets: []float64{6, 3, 3, 2, 4, 5, 1}, + CustomValues: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}, + } + + setup := func(h *histogram.FloatHistogram) (Chunk, *FloatHistogramAppender, int64, *histogram.FloatHistogram) { c := Chunk(NewFloatHistogramChunk()) // Create fresh appender and add the first histogram. @@ -289,32 +320,17 @@ func TestFloatHistogramChunkAppendable(t *testing.T) { require.Equal(t, 0, c.NumSamples()) ts := int64(1234567890) - h1 := &histogram.FloatHistogram{ - Count: 5, - ZeroCount: 2, - Sum: 18.4, - ZeroThreshold: 1e-125, - Schema: 1, - PositiveSpans: []histogram.Span{ - {Offset: 0, Length: 2}, - {Offset: 2, Length: 1}, - {Offset: 3, Length: 2}, - {Offset: 3, Length: 1}, - {Offset: 1, Length: 1}, - }, - PositiveBuckets: []float64{6, 3, 3, 2, 4, 5, 1}, - } - chk, _, app, err := app.AppendFloatHistogram(nil, ts, h1.Copy(), false) + chk, _, app, err := app.AppendFloatHistogram(nil, ts, h.Copy(), false) require.NoError(t, err) require.Nil(t, chk) require.Equal(t, 1, c.NumSamples()) require.Equal(t, UnknownCounterReset, c.(*FloatHistogramChunk).GetCounterResetHeader()) - return c, app.(*FloatHistogramAppender), ts, h1 + return c, app.(*FloatHistogramAppender), ts, h } { // Schema change. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.Schema++ _, _, ok, _ := hApp.appendable(h2) @@ -324,7 +340,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) { } { // Zero threshold change. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.ZeroThreshold += 0.1 _, _, ok, _ := hApp.appendable(h2) @@ -334,7 +350,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) { } { // New histogram that has more buckets. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.PositiveSpans = []histogram.Span{ {Offset: 0, Length: 3}, @@ -357,7 +373,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) { } { // New histogram that has a bucket missing. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.PositiveSpans = []histogram.Span{ @@ -379,7 +395,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) { } { // New histogram that has a counter reset while buckets are same. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.Sum = 23 h2.PositiveBuckets = []float64{6, 2, 3, 2, 4, 5, 1} @@ -394,7 +410,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) { } { // New histogram that has a counter reset while new buckets were added. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.PositiveSpans = []histogram.Span{ {Offset: 0, Length: 3}, @@ -415,7 +431,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) { } { - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) // New histogram that has a counter reset while new buckets were // added before the first bucket and reset on first bucket. (to // catch the edge case where the new bucket should be forwarded @@ -442,7 +458,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) { } { // New histogram that has an explicit counter reset. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.CounterResetHint = histogram.CounterReset @@ -450,7 +466,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) { } { // Start new chunk explicitly, and append a new histogram that is considered appendable to the previous chunk. - _, hApp, ts, h1 := setup() + _, hApp, ts, h1 := setup(eh) h2 := h1.Copy() // Identity is appendable. nextChunk := NewFloatHistogramChunk() @@ -466,7 +482,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) { } { // Start new chunk explicitly, and append a new histogram that is not considered appendable to the previous chunk. - _, hApp, ts, h1 := setup() + _, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.Count-- // Make this not appendable due to counter reset. @@ -483,7 +499,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) { } { // Start new chunk explicitly, and append a new histogram that would need recoding if we added it to the chunk. - _, hApp, ts, h1 := setup() + _, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.PositiveSpans = []histogram.Span{ {Offset: 0, Length: 3}, @@ -507,6 +523,72 @@ func TestFloatHistogramChunkAppendable(t *testing.T) { assertSampleCount(t, nextChunk, 1, ValFloatHistogram) require.Equal(t, NotCounterReset, nextChunk.GetCounterResetHeader()) } + + { // Custom buckets, no change. + c, hApp, ts, h1 := setup(cbh) + h2 := h1.Copy() + _, _, ok, _ := hApp.appendable(h2) + require.True(t, ok) + + assertNoNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset) + } + + { // Custom buckets, increase in bucket counts but no change in layout. + c, hApp, ts, h1 := setup(cbh) + h2 := h1.Copy() + h2.Count++ + h2.PositiveBuckets = []float64{6, 3, 3, 2, 4, 5, 2} + _, _, ok, _ := hApp.appendable(h2) + require.True(t, ok) + + assertNoNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset) + } + + { // Custom buckets, decrease in bucket counts but no change in layout. + c, hApp, ts, h1 := setup(cbh) + h2 := h1.Copy() + h2.Count-- + h2.PositiveBuckets = []float64{6, 3, 3, 2, 4, 5, 0} + _, _, ok, _ := hApp.appendable(h2) + require.False(t, ok) + + assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset) + } + + { // Custom buckets, change only in custom bounds. + c, hApp, ts, h1 := setup(cbh) + h2 := h1.Copy() + h2.CustomValues = []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21} + _, _, ok, _ := hApp.appendable(h2) + require.False(t, ok) + + assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset) + } + + { // Custom buckets, with more buckets. + c, hApp, ts, h1 := setup(cbh) + h2 := h1.Copy() + h2.PositiveSpans = []histogram.Span{ + {Offset: 0, Length: 3}, + {Offset: 1, Length: 1}, + {Offset: 1, Length: 4}, + {Offset: 3, Length: 3}, + } + h2.Count += 6 + h2.Sum = 30 + // Existing histogram should get values converted from the above to: + // 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between) + // so the new histogram should have new counts >= these per-bucket counts, e.g.: + h2.PositiveBuckets = []float64{7, 5, 1, 3, 1, 0, 2, 5, 5, 0, 1} // (total 30) + + posInterjections, negInterjections, ok, cr := hApp.appendable(h2) + require.NotEmpty(t, posInterjections) + require.Empty(t, negInterjections) + require.True(t, ok) // Only new buckets came in. + require.False(t, cr) + + assertRecodedFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset) + } } func assertNewFloatHistogramChunkOnAppend(t *testing.T, oldChunk Chunk, hApp *FloatHistogramAppender, ts int64, h *histogram.FloatHistogram, expectHeader CounterResetHeader) { @@ -526,7 +608,7 @@ func assertNewFloatHistogramChunkOnAppend(t *testing.T, oldChunk Chunk, hApp *Fl func assertNoNewFloatHistogramChunkOnAppend(t *testing.T, oldChunk Chunk, hApp *FloatHistogramAppender, ts int64, h *histogram.FloatHistogram, expectHeader CounterResetHeader) { oldChunkBytes := oldChunk.Bytes() newChunk, recoded, newAppender, err := hApp.AppendFloatHistogram(nil, ts, h, false) - require.NotEqual(t, oldChunkBytes, oldChunk.Bytes()) // Sanity check that previous chunk is untouched. + require.Greater(t, len(oldChunk.Bytes()), len(oldChunkBytes)) // Check that current chunk is bigger than previously. require.NoError(t, err) require.Nil(t, newChunk) require.False(t, recoded) @@ -715,6 +797,32 @@ func TestFloatHistogramChunkAppendableWithEmptySpan(t *testing.T) { NegativeBuckets: []float64{1, 4, 2, 7, 5, 5, 2}, }, }, + "empty span in old and new custom buckets histogram": { + h1: &histogram.FloatHistogram{ + Schema: histogram.CustomBucketsSchema, + Count: 7, + Sum: 1234.5, + PositiveSpans: []histogram.Span{ + {Offset: 0, Length: 4}, + {Offset: 0, Length: 0}, + {Offset: 0, Length: 3}, + }, + PositiveBuckets: []float64{1, 2, 1, 1, 1, 1, 1}, + CustomValues: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + }, + h2: &histogram.FloatHistogram{ + Schema: histogram.CustomBucketsSchema, + Count: 10, + Sum: 2345.6, + PositiveSpans: []histogram.Span{ + {Offset: 0, Length: 4}, + {Offset: 0, Length: 0}, + {Offset: 0, Length: 3}, + }, + PositiveBuckets: []float64{1, 3, 1, 2, 1, 1, 1}, + CustomValues: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + }, + }, } for name, tc := range tests { @@ -741,7 +849,40 @@ func TestFloatHistogramChunkAppendableWithEmptySpan(t *testing.T) { } func TestFloatHistogramChunkAppendableGauge(t *testing.T) { - setup := func() (Chunk, *FloatHistogramAppender, int64, *histogram.FloatHistogram) { + eh := &histogram.FloatHistogram{ + CounterResetHint: histogram.GaugeType, + Count: 5, + ZeroCount: 2, + Sum: 18.4, + ZeroThreshold: 1e-125, + Schema: 1, + PositiveSpans: []histogram.Span{ + {Offset: 0, Length: 2}, + {Offset: 2, Length: 1}, + {Offset: 3, Length: 2}, + {Offset: 3, Length: 1}, + {Offset: 1, Length: 1}, + }, + PositiveBuckets: []float64{6, 3, 3, 2, 4, 5, 1}, + } + + cbh := &histogram.FloatHistogram{ + CounterResetHint: histogram.GaugeType, + Count: 24, + Sum: 18.4, + Schema: histogram.CustomBucketsSchema, + PositiveSpans: []histogram.Span{ + {Offset: 0, Length: 2}, + {Offset: 2, Length: 1}, + {Offset: 3, Length: 2}, + {Offset: 3, Length: 1}, + {Offset: 1, Length: 1}, + }, + PositiveBuckets: []float64{6, 3, 3, 2, 4, 5, 1}, + CustomValues: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}, + } + + setup := func(h *histogram.FloatHistogram) (Chunk, *FloatHistogramAppender, int64, *histogram.FloatHistogram) { c := Chunk(NewFloatHistogramChunk()) // Create fresh appender and add the first histogram. @@ -750,33 +891,17 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) { require.Equal(t, 0, c.NumSamples()) ts := int64(1234567890) - h1 := &histogram.FloatHistogram{ - CounterResetHint: histogram.GaugeType, - Count: 5, - ZeroCount: 2, - Sum: 18.4, - ZeroThreshold: 1e-125, - Schema: 1, - PositiveSpans: []histogram.Span{ - {Offset: 0, Length: 2}, - {Offset: 2, Length: 1}, - {Offset: 3, Length: 2}, - {Offset: 3, Length: 1}, - {Offset: 1, Length: 1}, - }, - PositiveBuckets: []float64{6, 3, 3, 2, 4, 5, 1}, - } - chk, _, app, err := app.AppendFloatHistogram(nil, ts, h1.Copy(), false) + chk, _, app, err := app.AppendFloatHistogram(nil, ts, h.Copy(), false) require.NoError(t, err) require.Nil(t, chk) require.Equal(t, 1, c.NumSamples()) require.Equal(t, GaugeType, c.(*FloatHistogramChunk).GetCounterResetHeader()) - return c, app.(*FloatHistogramAppender), ts, h1 + return c, app.(*FloatHistogramAppender), ts, h } { // Schema change. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.Schema++ _, _, _, _, _, _, ok := hApp.appendableGauge(h2) @@ -786,7 +911,7 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) { } { // Zero threshold change. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.ZeroThreshold += 0.1 _, _, _, _, _, _, ok := hApp.appendableGauge(h2) @@ -796,7 +921,7 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) { } { // New histogram that has more buckets. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.PositiveSpans = []histogram.Span{ {Offset: 0, Length: 3}, @@ -820,7 +945,7 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) { } { // New histogram that has buckets missing. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.PositiveSpans = []histogram.Span{ {Offset: 0, Length: 2}, @@ -844,7 +969,7 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) { } { // New histogram that has a bucket missing and new buckets. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.PositiveSpans = []histogram.Span{ {Offset: 0, Length: 2}, @@ -866,7 +991,7 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) { } { // New histogram that has a counter reset while buckets are same. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.Sum = 23 h2.PositiveBuckets = []float64{6, 2, 3, 2, 4, 5, 1} @@ -882,7 +1007,7 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) { } { // New histogram that has a counter reset while new buckets were added. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.PositiveSpans = []histogram.Span{ {Offset: 0, Length: 3}, @@ -906,7 +1031,7 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) { { // New histogram that has a counter reset while new buckets were // added before the first bucket and reset on first bucket. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.PositiveSpans = []histogram.Span{ {Offset: -3, Length: 2}, @@ -928,6 +1053,73 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) { assertRecodedFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType) } + + { // Custom buckets, no change. + c, hApp, ts, h1 := setup(cbh) + h2 := h1.Copy() + _, _, _, _, _, _, ok := hApp.appendableGauge(h2) + require.True(t, ok) + + assertNoNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType) + } + + { // Custom buckets, increase in bucket counts but no change in layout. + c, hApp, ts, h1 := setup(cbh) + h2 := h1.Copy() + h2.Count++ + h2.PositiveBuckets = []float64{6, 3, 3, 2, 4, 5, 2} + _, _, _, _, _, _, ok := hApp.appendableGauge(h2) + require.True(t, ok) + + assertNoNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType) + } + + { // Custom buckets, decrease in bucket counts but no change in layout. + c, hApp, ts, h1 := setup(cbh) + h2 := h1.Copy() + h2.Count-- + h2.PositiveBuckets = []float64{6, 3, 3, 2, 4, 5, 0} + _, _, _, _, _, _, ok := hApp.appendableGauge(h2) + require.True(t, ok) + + assertNoNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType) + } + + { // Custom buckets, change only in custom bounds. + c, hApp, ts, h1 := setup(cbh) + h2 := h1.Copy() + h2.CustomValues = []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21} + _, _, _, _, _, _, ok := hApp.appendableGauge(h2) + require.False(t, ok) + + assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType) + } + + { // Custom buckets, with more buckets. + c, hApp, ts, h1 := setup(cbh) + h2 := h1.Copy() + h2.PositiveSpans = []histogram.Span{ + {Offset: 0, Length: 3}, + {Offset: 1, Length: 1}, + {Offset: 1, Length: 4}, + {Offset: 3, Length: 3}, + } + h2.Count += 6 + h2.Sum = 30 + // Existing histogram should get values converted from the above to: + // 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between) + // so the new histogram should have new counts >= these per-bucket counts, e.g.: + h2.PositiveBuckets = []float64{7, 5, 1, 3, 1, 0, 2, 5, 5, 0, 1} // (total 30) + + posInterjections, negInterjections, pBackwardI, nBackwardI, _, _, ok := hApp.appendableGauge(h2) + require.NotEmpty(t, posInterjections) + require.Empty(t, negInterjections) + require.Empty(t, pBackwardI) + require.Empty(t, nBackwardI) + require.True(t, ok) // Only new buckets came in. + + assertRecodedFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType) + } } func TestFloatHistogramAppendOnlyErrors(t *testing.T) { @@ -975,4 +1167,26 @@ func TestFloatHistogramAppendOnlyErrors(t *testing.T) { require.False(t, isRecoded) require.EqualError(t, err, "float histogram counter reset") }) + t.Run("counter reset error with custom buckets", func(t *testing.T) { + c := Chunk(NewFloatHistogramChunk()) + + // Create fresh appender and add the first histogram. + app, err := c.Appender() + require.NoError(t, err) + + h := tsdbutil.GenerateTestCustomBucketsFloatHistogram(0) + var isRecoded bool + c, isRecoded, app, err = app.AppendFloatHistogram(nil, 1, h, true) + require.Nil(t, c) + require.False(t, isRecoded) + require.NoError(t, err) + + // Add erroring histogram. + h2 := h.Copy() + h2.CustomValues = []float64{0, 1, 2, 3, 4, 5, 6, 7} + c, isRecoded, _, err = app.AppendFloatHistogram(nil, 2, h2, true) + require.Nil(t, c) + require.False(t, isRecoded) + require.EqualError(t, err, "float histogram counter reset") + }) } diff --git a/tsdb/chunkenc/histogram.go b/tsdb/chunkenc/histogram.go index cb09eda26d..48c13a3ac4 100644 --- a/tsdb/chunkenc/histogram.go +++ b/tsdb/chunkenc/histogram.go @@ -65,6 +65,7 @@ func (c *HistogramChunk) NumSamples() int { func (c *HistogramChunk) Layout() ( schema int32, zeroThreshold float64, negativeSpans, positiveSpans []histogram.Span, + customValues []float64, err error, ) { if c.NumSamples() == 0 { @@ -127,6 +128,7 @@ func (c *HistogramChunk) Appender() (Appender, error) { zThreshold: it.zThreshold, pSpans: it.pSpans, nSpans: it.nSpans, + customValues: it.customValues, t: it.t, cnt: it.cnt, zCnt: it.zCnt, @@ -194,6 +196,7 @@ type HistogramAppender struct { schema int32 zThreshold float64 pSpans, nSpans []histogram.Span + customValues []float64 // Although we intend to start new chunks on counter resets, we still // have to handle negative deltas for gauge histograms. Therefore, even @@ -237,6 +240,7 @@ func (a *HistogramAppender) Append(int64, float64) { // The chunk is not appendable in the following cases: // // - The schema has changed. +// - The custom bounds have changed if the current schema is custom buckets. // - The threshold for the zero bucket has changed. // - Any buckets have disappeared. // - There was a counter reset in the count of observations or in any bucket, @@ -279,6 +283,11 @@ func (a *HistogramAppender) appendable(h *histogram.Histogram) ( return } + if histogram.IsCustomBucketsSchema(h.Schema) && !histogram.FloatBucketsMatch(h.CustomValues, a.customValues) { + counterReset = true + return + } + if h.ZeroCount < a.zCnt { // There has been a counter reset since ZeroThreshold didn't change. counterReset = true @@ -319,6 +328,7 @@ func (a *HistogramAppender) appendable(h *histogram.Histogram) ( // // The chunk is not appendable in the following cases: // - The schema has changed. +// - The custom bounds have changed if the current schema is custom buckets. // - The threshold for the zero bucket has changed. // - The last sample in the chunk was stale while the current sample is not stale. func (a *HistogramAppender) appendableGauge(h *histogram.Histogram) ( @@ -345,6 +355,10 @@ func (a *HistogramAppender) appendableGauge(h *histogram.Histogram) ( return } + if histogram.IsCustomBucketsSchema(h.Schema) && !histogram.FloatBucketsMatch(h.CustomValues, a.customValues) { + return + } + positiveInserts, backwardPositiveInserts, positiveSpans = expandSpansBothWays(a.pSpans, h.PositiveSpans) negativeInserts, backwardNegativeInserts, negativeSpans = expandSpansBothWays(a.nSpans, h.NegativeSpans) okToAppend = true @@ -438,7 +452,7 @@ func (a *HistogramAppender) appendHistogram(t int64, h *histogram.Histogram) { if num == 0 { // The first append gets the privilege to dictate the layout // but it's also responsible for encoding it into the chunk! - writeHistogramChunkLayout(a.b, h.Schema, h.ZeroThreshold, h.PositiveSpans, h.NegativeSpans) + writeHistogramChunkLayout(a.b, h.Schema, h.ZeroThreshold, h.PositiveSpans, h.NegativeSpans, h.CustomValues) a.schema = h.Schema a.zThreshold = h.ZeroThreshold @@ -454,6 +468,12 @@ func (a *HistogramAppender) appendHistogram(t int64, h *histogram.Histogram) { } else { a.nSpans = nil } + if len(h.CustomValues) > 0 { + a.customValues = make([]float64, len(h.CustomValues)) + copy(a.customValues, h.CustomValues) + } else { + a.customValues = nil + } numPBuckets, numNBuckets := countSpans(h.PositiveSpans), countSpans(h.NegativeSpans) if numPBuckets > 0 { @@ -737,6 +757,7 @@ type histogramIterator struct { schema int32 zThreshold float64 pSpans, nSpans []histogram.Span + customValues []float64 // For the fields that are tracked as deltas and ultimately dod's. t int64 @@ -793,6 +814,7 @@ func (it *histogramIterator) AtHistogram(h *histogram.Histogram) (int64, *histog NegativeSpans: it.nSpans, PositiveBuckets: it.pBuckets, NegativeBuckets: it.nBuckets, + CustomValues: it.customValues, } } @@ -815,6 +837,9 @@ func (it *histogramIterator) AtHistogram(h *histogram.Histogram) (int64, *histog h.NegativeBuckets = resize(h.NegativeBuckets, len(it.nBuckets)) copy(h.NegativeBuckets, it.nBuckets) + h.CustomValues = resize(h.CustomValues, len(it.customValues)) + copy(h.CustomValues, it.customValues) + return it.t, h } @@ -835,6 +860,7 @@ func (it *histogramIterator) AtFloatHistogram(fh *histogram.FloatHistogram) (int NegativeSpans: it.nSpans, PositiveBuckets: it.pFloatBuckets, NegativeBuckets: it.nFloatBuckets, + CustomValues: it.customValues, } } @@ -865,6 +891,9 @@ func (it *histogramIterator) AtFloatHistogram(fh *histogram.FloatHistogram) (int fh.NegativeBuckets[i] = currentNegative } + fh.CustomValues = resize(fh.CustomValues, len(it.customValues)) + copy(fh.CustomValues, it.customValues) + return it.t, fh } @@ -923,7 +952,7 @@ func (it *histogramIterator) Next() ValueType { // The first read is responsible for reading the chunk layout // and for initializing fields that depend on it. We give // counter reset info at chunk level, hence we discard it here. - schema, zeroThreshold, posSpans, negSpans, err := readHistogramChunkLayout(&it.br) + schema, zeroThreshold, posSpans, negSpans, customValues, err := readHistogramChunkLayout(&it.br) if err != nil { it.err = err return ValNone @@ -931,6 +960,7 @@ func (it *histogramIterator) Next() ValueType { it.schema = schema it.zThreshold = zeroThreshold it.pSpans, it.nSpans = posSpans, negSpans + it.customValues = customValues numPBuckets, numNBuckets := countSpans(posSpans), countSpans(negSpans) // The code below recycles existing slices in case this iterator // was reset and already has slices of a sufficient capacity. diff --git a/tsdb/chunkenc/histogram_meta.go b/tsdb/chunkenc/histogram_meta.go index 70f129b953..9b65a05ba5 100644 --- a/tsdb/chunkenc/histogram_meta.go +++ b/tsdb/chunkenc/histogram_meta.go @@ -21,17 +21,21 @@ import ( func writeHistogramChunkLayout( b *bstream, schema int32, zeroThreshold float64, - positiveSpans, negativeSpans []histogram.Span, + positiveSpans, negativeSpans []histogram.Span, customValues []float64, ) { putZeroThreshold(b, zeroThreshold) putVarbitInt(b, int64(schema)) putHistogramChunkLayoutSpans(b, positiveSpans) putHistogramChunkLayoutSpans(b, negativeSpans) + if histogram.IsCustomBucketsSchema(schema) { + putHistogramChunkLayoutCustomBounds(b, customValues) + } } func readHistogramChunkLayout(b *bstreamReader) ( schema int32, zeroThreshold float64, positiveSpans, negativeSpans []histogram.Span, + customValues []float64, err error, ) { zeroThreshold, err = readZeroThreshold(b) @@ -55,6 +59,13 @@ func readHistogramChunkLayout(b *bstreamReader) ( return } + if histogram.IsCustomBucketsSchema(schema) { + customValues, err = readHistogramChunkLayoutCustomBounds(b) + if err != nil { + return + } + } + return } @@ -92,6 +103,30 @@ func readHistogramChunkLayoutSpans(b *bstreamReader) ([]histogram.Span, error) { return spans, nil } +func putHistogramChunkLayoutCustomBounds(b *bstream, customValues []float64) { + putVarbitUint(b, uint64(len(customValues))) + for _, bound := range customValues { + putCustomBound(b, bound) + } +} + +func readHistogramChunkLayoutCustomBounds(b *bstreamReader) ([]float64, error) { + var customValues []float64 + num, err := readVarbitUint(b) + if err != nil { + return nil, err + } + for i := 0; i < int(num); i++ { + bound, err := readCustomBound(b) + if err != nil { + return nil, err + } + + customValues = append(customValues, bound) + } + return customValues, nil +} + // putZeroThreshold writes the zero threshold to the bstream. It stores typical // values in just one byte, but needs 9 bytes for other values. In detail: // - If the threshold is 0, store a single zero byte. @@ -140,6 +175,52 @@ func readZeroThreshold(br *bstreamReader) (float64, error) { } } +// isWholeWhenMultiplied checks to see if the number when multiplied by 1000 can +// be converted into an integer without losing precision. +func isWholeWhenMultiplied(in float64) bool { + i := uint(math.Round(in * 1000)) + out := float64(i) / 1000 + return in == out +} + +// putCustomBound writes the custom bound to the bstream. It stores values from 0 to +// 16.382 (inclusive) that are multiples of 0.001 in an unsigned var int of up to 2 bytes, +// but needs 1 bit + 8 bytes for other values like negative numbers, numbers greater than +// 16.382, or numbers that are not a multiple of 0.001, on the assumption that they are +// less common. In detail: +// - Multiply the bound by 1000, without rounding. +// - If the multiplied bound is >= 0, <= 16382 and a whole number, store it as an +// unsigned var int. +// - Otherwise, store 0 as an unsigned var int, followed by the 8 bytes of the original +// bound as a float64. +func putCustomBound(b *bstream, f float64) { + tf := f * 1000 + if tf < 0 || tf > 16382 || !isWholeWhenMultiplied(f) { + b.putUvarint(0) + b.writeBits(math.Float64bits(f), 64) + return + } + b.putUvarint(uint64(math.Round(tf) + 1)) +} + +// readCustomBound reads the custom bound written with putCustomBound. +func readCustomBound(br *bstreamReader) (float64, error) { + b, err := br.readUvarint() + if err != nil { + return 0, err + } + switch b { + case 0: + v, err := br.readBits(64) + if err != nil { + return 0, err + } + return math.Float64frombits(v), nil + default: + return float64(b-1) / 1000, nil + } +} + type bucketIterator struct { spans []histogram.Span span int // Span position of last yielded bucket. diff --git a/tsdb/chunkenc/histogram_meta_test.go b/tsdb/chunkenc/histogram_meta_test.go index 0b2b187475..c3ff4aabc2 100644 --- a/tsdb/chunkenc/histogram_meta_test.go +++ b/tsdb/chunkenc/histogram_meta_test.go @@ -373,6 +373,7 @@ func TestWriteReadHistogramChunkLayout(t *testing.T) { schema int32 zeroThreshold float64 positiveSpans, negativeSpans []histogram.Span + customValues []float64 }{ { schema: 3, @@ -422,23 +423,48 @@ func TestWriteReadHistogramChunkLayout(t *testing.T) { positiveSpans: nil, negativeSpans: nil, }, + { + schema: histogram.CustomBucketsSchema, + positiveSpans: []histogram.Span{{Offset: -4, Length: 3}, {Offset: 2, Length: 42}}, + negativeSpans: nil, + customValues: []float64{-5, -2.5, 0, 0.1, 0.25, 0.5, 1, 2, 5, 10, 25, 50, 100, 255, 500, 1000}, + }, + { + schema: histogram.CustomBucketsSchema, + positiveSpans: []histogram.Span{{Offset: -4, Length: 3}, {Offset: 2, Length: 42}}, + negativeSpans: nil, + customValues: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 25.0, 50.0, 100.0}, + }, + { + schema: histogram.CustomBucketsSchema, + positiveSpans: []histogram.Span{{Offset: -4, Length: 3}, {Offset: 2, Length: 42}}, + negativeSpans: nil, + customValues: []float64{0.001, 0.002, 0.004, 0.008, 0.016, 0.032, 0.064, 0.128, 0.256, 0.512, 1.024, 2.048, 4.096, 8.192}, + }, + { + schema: histogram.CustomBucketsSchema, + positiveSpans: []histogram.Span{{Offset: -4, Length: 3}, {Offset: 2, Length: 42}}, + negativeSpans: nil, + customValues: []float64{1.001, 1.023, 2.01, 4.007, 4.095, 8.001, 8.19, 16.24}, + }, } bs := bstream{} for _, l := range layouts { - writeHistogramChunkLayout(&bs, l.schema, l.zeroThreshold, l.positiveSpans, l.negativeSpans) + writeHistogramChunkLayout(&bs, l.schema, l.zeroThreshold, l.positiveSpans, l.negativeSpans, l.customValues) } bsr := newBReader(bs.bytes()) for _, want := range layouts { - gotSchema, gotZeroThreshold, gotPositiveSpans, gotNegativeSpans, err := readHistogramChunkLayout(&bsr) + gotSchema, gotZeroThreshold, gotPositiveSpans, gotNegativeSpans, gotCustomBounds, err := readHistogramChunkLayout(&bsr) require.NoError(t, err) require.Equal(t, want.schema, gotSchema) require.Equal(t, want.zeroThreshold, gotZeroThreshold) require.Equal(t, want.positiveSpans, gotPositiveSpans) require.Equal(t, want.negativeSpans, gotNegativeSpans) + require.Equal(t, want.customValues, gotCustomBounds) } } diff --git a/tsdb/chunkenc/histogram_test.go b/tsdb/chunkenc/histogram_test.go index f7609c1936..d029aaefcb 100644 --- a/tsdb/chunkenc/histogram_test.go +++ b/tsdb/chunkenc/histogram_test.go @@ -294,7 +294,38 @@ func TestHistogramChunkBucketChanges(t *testing.T) { } func TestHistogramChunkAppendable(t *testing.T) { - setup := func() (Chunk, *HistogramAppender, int64, *histogram.Histogram) { + eh := &histogram.Histogram{ + Count: 5, + ZeroCount: 2, + Sum: 18.4, + ZeroThreshold: 1e-125, + Schema: 1, + PositiveSpans: []histogram.Span{ + {Offset: 0, Length: 2}, + {Offset: 2, Length: 1}, + {Offset: 3, Length: 2}, + {Offset: 3, Length: 1}, + {Offset: 1, Length: 1}, + }, + PositiveBuckets: []int64{6, -3, 0, -1, 2, 1, -4}, // counts: 6, 3, 3, 2, 4, 5, 1 (total 24) + } + + cbh := &histogram.Histogram{ + Count: 24, + Sum: 18.4, + Schema: histogram.CustomBucketsSchema, + PositiveSpans: []histogram.Span{ + {Offset: 0, Length: 2}, + {Offset: 2, Length: 1}, + {Offset: 3, Length: 2}, + {Offset: 3, Length: 1}, + {Offset: 1, Length: 1}, + }, + PositiveBuckets: []int64{6, -3, 0, -1, 2, 1, -4}, // counts: 6, 3, 3, 2, 4, 5, 1 (total 24) + CustomValues: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}, + } + + setup := func(h *histogram.Histogram) (Chunk, *HistogramAppender, int64, *histogram.Histogram) { c := Chunk(NewHistogramChunk()) // Create fresh appender and add the first histogram. @@ -303,32 +334,17 @@ func TestHistogramChunkAppendable(t *testing.T) { require.Equal(t, 0, c.NumSamples()) ts := int64(1234567890) - h1 := &histogram.Histogram{ - Count: 5, - ZeroCount: 2, - Sum: 18.4, - ZeroThreshold: 1e-125, - Schema: 1, - PositiveSpans: []histogram.Span{ - {Offset: 0, Length: 2}, - {Offset: 2, Length: 1}, - {Offset: 3, Length: 2}, - {Offset: 3, Length: 1}, - {Offset: 1, Length: 1}, - }, - PositiveBuckets: []int64{6, -3, 0, -1, 2, 1, -4}, // counts: 6, 3, 3, 2, 4, 5, 1 (total 24) - } - chk, _, app, err := app.AppendHistogram(nil, ts, h1.Copy(), false) + chk, _, app, err := app.AppendHistogram(nil, ts, h.Copy(), false) require.NoError(t, err) require.Nil(t, chk) require.Equal(t, 1, c.NumSamples()) require.Equal(t, UnknownCounterReset, c.(*HistogramChunk).GetCounterResetHeader()) - return c, app.(*HistogramAppender), ts, h1 + return c, app.(*HistogramAppender), ts, h } { // Schema change. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.Schema++ _, _, ok, _ := hApp.appendable(h2) @@ -338,7 +354,7 @@ func TestHistogramChunkAppendable(t *testing.T) { } { // Zero threshold change. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.ZeroThreshold += 0.1 _, _, ok, _ := hApp.appendable(h2) @@ -348,7 +364,7 @@ func TestHistogramChunkAppendable(t *testing.T) { } { // New histogram that has more buckets. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.PositiveSpans = []histogram.Span{ {Offset: 0, Length: 3}, @@ -374,7 +390,7 @@ func TestHistogramChunkAppendable(t *testing.T) { } { // New histogram that has a bucket missing. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.PositiveSpans = []histogram.Span{ {Offset: 0, Length: 2}, @@ -395,7 +411,7 @@ func TestHistogramChunkAppendable(t *testing.T) { } { // New histogram that has a counter reset while buckets are same. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.Sum = 23 h2.PositiveBuckets = []int64{6, -4, 1, -1, 2, 1, -4} // counts: 6, 2, 3, 2, 4, 5, 1 (total 23) @@ -410,7 +426,7 @@ func TestHistogramChunkAppendable(t *testing.T) { } { // New histogram that has a counter reset while new buckets were added. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.PositiveSpans = []histogram.Span{ {Offset: 0, Length: 3}, @@ -438,7 +454,7 @@ func TestHistogramChunkAppendable(t *testing.T) { // added before the first bucket and reset on first bucket. (to // catch the edge case where the new bucket should be forwarded // ahead until first old bucket at start) - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.PositiveSpans = []histogram.Span{ {Offset: -3, Length: 2}, @@ -464,7 +480,7 @@ func TestHistogramChunkAppendable(t *testing.T) { } { // New histogram that has an explicit counter reset. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.CounterResetHint = histogram.CounterReset @@ -472,7 +488,7 @@ func TestHistogramChunkAppendable(t *testing.T) { } { // Start new chunk explicitly, and append a new histogram that is considered appendable to the previous chunk. - _, hApp, ts, h1 := setup() + _, hApp, ts, h1 := setup(eh) h2 := h1.Copy() // Identity is appendable. nextChunk := NewHistogramChunk() @@ -488,7 +504,7 @@ func TestHistogramChunkAppendable(t *testing.T) { } { // Start new chunk explicitly, and append a new histogram that is not considered appendable to the previous chunk. - _, hApp, ts, h1 := setup() + _, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.Count-- // Make this not appendable due to counter reset. @@ -505,7 +521,7 @@ func TestHistogramChunkAppendable(t *testing.T) { } { // Start new chunk explicitly, and append a new histogram that would need recoding if we added it to the chunk. - _, hApp, ts, h1 := setup() + _, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.PositiveSpans = []histogram.Span{ {Offset: 0, Length: 3}, @@ -532,6 +548,72 @@ func TestHistogramChunkAppendable(t *testing.T) { assertSampleCount(t, nextChunk, 1, ValHistogram) require.Equal(t, NotCounterReset, nextChunk.GetCounterResetHeader()) } + + { // Custom buckets, no change. + c, hApp, ts, h1 := setup(cbh) + h2 := h1.Copy() + _, _, ok, _ := hApp.appendable(h2) + require.True(t, ok) + + assertNoNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset) + } + + { // Custom buckets, increase in bucket counts but no change in layout. + c, hApp, ts, h1 := setup(cbh) + h2 := h1.Copy() + h2.Count++ + h2.PositiveBuckets = []int64{6, -3, 0, -1, 2, 1, -3} + _, _, ok, _ := hApp.appendable(h2) + require.True(t, ok) + + assertNoNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset) + } + + { // Custom buckets, decrease in bucket counts but no change in layout. + c, hApp, ts, h1 := setup(cbh) + h2 := h1.Copy() + h2.Count-- + h2.PositiveBuckets = []int64{6, -3, 0, -1, 2, 1, -5} + _, _, ok, _ := hApp.appendable(h2) + require.False(t, ok) + + assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset) + } + + { // Custom buckets, change only in custom bounds. + c, hApp, ts, h1 := setup(cbh) + h2 := h1.Copy() + h2.CustomValues = []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21} + _, _, ok, _ := hApp.appendable(h2) + require.False(t, ok) + + assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset) + } + + { // Custom buckets, with more buckets. + c, hApp, ts, h1 := setup(cbh) + h2 := h1.Copy() + h2.PositiveSpans = []histogram.Span{ + {Offset: 0, Length: 3}, + {Offset: 1, Length: 1}, + {Offset: 1, Length: 4}, + {Offset: 3, Length: 3}, + } + h2.Count += 6 + h2.Sum = 30 + // Existing histogram should get values converted from the above to: + // 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between) + // so the new histogram should have new counts >= these per-bucket counts, e.g.: + h2.PositiveBuckets = []int64{7, -2, -4, 2, -2, -1, 2, 3, 0, -5, 1} // 7 5 1 3 1 0 2 5 5 0 1 (total 30) + + posInterjections, negInterjections, ok, cr := hApp.appendable(h2) + require.NotEmpty(t, posInterjections) + require.Empty(t, negInterjections) + require.True(t, ok) // Only new buckets came in. + require.False(t, cr) + + assertRecodedHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset) + } } func assertNewHistogramChunkOnAppend(t *testing.T, oldChunk Chunk, hApp *HistogramAppender, ts int64, h *histogram.Histogram, expectHeader CounterResetHeader) { @@ -548,6 +630,19 @@ func assertNewHistogramChunkOnAppend(t *testing.T, oldChunk Chunk, hApp *Histogr assertSampleCount(t, newChunk, 1, ValHistogram) } +func assertNoNewHistogramChunkOnAppend(t *testing.T, currChunk Chunk, hApp *HistogramAppender, ts int64, h *histogram.Histogram, expectHeader CounterResetHeader) { + prevChunkBytes := currChunk.Bytes() + newChunk, recoded, newAppender, err := hApp.AppendHistogram(nil, ts, h, false) + require.Greater(t, len(currChunk.Bytes()), len(prevChunkBytes)) // Check that current chunk is bigger than previously. + require.NoError(t, err) + require.Nil(t, newChunk) + require.False(t, recoded) + require.Equal(t, expectHeader, currChunk.(*HistogramChunk).GetCounterResetHeader()) + require.NotNil(t, newAppender) + require.Equal(t, hApp, newAppender) + assertSampleCount(t, currChunk, 2, ValHistogram) +} + func assertRecodedHistogramChunkOnAppend(t *testing.T, prevChunk Chunk, hApp *HistogramAppender, ts int64, h *histogram.Histogram, expectHeader CounterResetHeader) { prevChunkBytes := prevChunk.Bytes() newChunk, recoded, newAppender, err := hApp.AppendHistogram(nil, ts, h, false) @@ -738,6 +833,32 @@ func TestHistogramChunkAppendableWithEmptySpan(t *testing.T) { NegativeBuckets: []int64{1, 3, -2, 5, -2, 0, -3}, }, }, + "empty span in old and new custom buckets histogram": { + h1: &histogram.Histogram{ + Schema: histogram.CustomBucketsSchema, + Count: 7, + Sum: 1234.5, + PositiveSpans: []histogram.Span{ + {Offset: 0, Length: 4}, + {Offset: 0, Length: 0}, + {Offset: 0, Length: 3}, + }, + PositiveBuckets: []int64{1, 1, -1, 0, 0, 0, 0}, + CustomValues: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + }, + h2: &histogram.Histogram{ + Schema: histogram.CustomBucketsSchema, + Count: 10, + Sum: 2345.6, + PositiveSpans: []histogram.Span{ + {Offset: 0, Length: 4}, + {Offset: 0, Length: 0}, + {Offset: 0, Length: 3}, + }, + PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 0}, + CustomValues: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + }, + }, } for name, tc := range tests { @@ -905,7 +1026,40 @@ func TestAtFloatHistogram(t *testing.T) { } func TestHistogramChunkAppendableGauge(t *testing.T) { - setup := func() (Chunk, *HistogramAppender, int64, *histogram.Histogram) { + eh := &histogram.Histogram{ + CounterResetHint: histogram.GaugeType, + Count: 5, + ZeroCount: 2, + Sum: 18.4, + ZeroThreshold: 1e-125, + Schema: 1, + PositiveSpans: []histogram.Span{ + {Offset: 0, Length: 2}, + {Offset: 2, Length: 1}, + {Offset: 3, Length: 2}, + {Offset: 3, Length: 1}, + {Offset: 1, Length: 1}, + }, + PositiveBuckets: []int64{6, -3, 0, -1, 2, 1, -4}, // {6, 3, 3, 2, 4, 5, 1} + } + + cbh := &histogram.Histogram{ + CounterResetHint: histogram.GaugeType, + Count: 24, + Sum: 18.4, + Schema: histogram.CustomBucketsSchema, + PositiveSpans: []histogram.Span{ + {Offset: 0, Length: 2}, + {Offset: 2, Length: 1}, + {Offset: 3, Length: 2}, + {Offset: 3, Length: 1}, + {Offset: 1, Length: 1}, + }, + PositiveBuckets: []int64{6, -3, 0, -1, 2, 1, -4}, // {6, 3, 3, 2, 4, 5, 1} + CustomValues: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}, + } + + setup := func(h *histogram.Histogram) (Chunk, *HistogramAppender, int64, *histogram.Histogram) { c := Chunk(NewHistogramChunk()) // Create fresh appender and add the first histogram. @@ -914,66 +1068,38 @@ func TestHistogramChunkAppendableGauge(t *testing.T) { require.Equal(t, 0, c.NumSamples()) ts := int64(1234567890) - h1 := &histogram.Histogram{ - CounterResetHint: histogram.GaugeType, - Count: 5, - ZeroCount: 2, - Sum: 18.4, - ZeroThreshold: 1e-125, - Schema: 1, - PositiveSpans: []histogram.Span{ - {Offset: 0, Length: 2}, - {Offset: 2, Length: 1}, - {Offset: 3, Length: 2}, - {Offset: 3, Length: 1}, - {Offset: 1, Length: 1}, - }, - PositiveBuckets: []int64{6, -3, 0, -1, 2, 1, -4}, // {6, 3, 3, 2, 4, 5, 1} - } - chk, _, app, err := app.AppendHistogram(nil, ts, h1.Copy(), false) + chk, _, app, err := app.AppendHistogram(nil, ts, h.Copy(), false) require.NoError(t, err) require.Nil(t, chk) require.Equal(t, 1, c.NumSamples()) require.Equal(t, GaugeType, c.(*HistogramChunk).GetCounterResetHeader()) - return c, app.(*HistogramAppender), ts, h1 + return c, app.(*HistogramAppender), ts, h } { // Schema change. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.Schema++ _, _, _, _, _, _, ok := hApp.appendableGauge(h2) require.False(t, ok) - newc, recoded, _, err := hApp.AppendHistogram(nil, ts+1, h2, false) - require.NoError(t, err) - require.NotNil(t, newc) - require.False(t, recoded) - require.NotEqual(t, c, newc) - require.Equal(t, GaugeType, c.(*HistogramChunk).GetCounterResetHeader()) - require.Equal(t, GaugeType, newc.(*HistogramChunk).GetCounterResetHeader()) + assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType) } { // Zero threshold change. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.ZeroThreshold += 0.1 _, _, _, _, _, _, ok := hApp.appendableGauge(h2) require.False(t, ok) - newc, recoded, _, err := hApp.AppendHistogram(nil, ts+1, h2, false) - require.NoError(t, err) - require.NotNil(t, newc) - require.False(t, recoded) - require.NotEqual(t, c, newc) - require.Equal(t, GaugeType, c.(*HistogramChunk).GetCounterResetHeader()) - require.Equal(t, GaugeType, newc.(*HistogramChunk).GetCounterResetHeader()) + assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType) } { // New histogram that has more buckets. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.PositiveSpans = []histogram.Span{ {Offset: 0, Length: 3}, @@ -993,15 +1119,11 @@ func TestHistogramChunkAppendableGauge(t *testing.T) { require.Empty(t, nBackwardI) require.True(t, ok) - newc, recoded, _, err := hApp.AppendHistogram(nil, ts+1, h2, false) - require.NoError(t, err) - require.NotNil(t, newc) - require.True(t, recoded) - require.Equal(t, GaugeType, c.(*HistogramChunk).GetCounterResetHeader()) + assertRecodedHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType) } { // New histogram that has buckets missing. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.PositiveSpans = []histogram.Span{ {Offset: 0, Length: 2}, @@ -1021,15 +1143,11 @@ func TestHistogramChunkAppendableGauge(t *testing.T) { require.Empty(t, nBackwardI) require.True(t, ok) - newc, recoded, _, err := hApp.AppendHistogram(nil, ts+1, h2, false) - require.NoError(t, err) - require.Nil(t, newc) - require.False(t, recoded) - require.Equal(t, GaugeType, c.(*HistogramChunk).GetCounterResetHeader()) + assertNoNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType) } { // New histogram that has a bucket missing and new buckets. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.PositiveSpans = []histogram.Span{ {Offset: 0, Length: 2}, @@ -1047,15 +1165,11 @@ func TestHistogramChunkAppendableGauge(t *testing.T) { require.Empty(t, nBackwardI) require.True(t, ok) - newc, recoded, _, err := hApp.AppendHistogram(nil, ts+1, h2, false) - require.NoError(t, err) - require.NotNil(t, newc) - require.True(t, recoded) - require.Equal(t, GaugeType, c.(*HistogramChunk).GetCounterResetHeader()) + assertRecodedHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType) } { // New histogram that has a counter reset while buckets are same. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.Sum = 23 h2.PositiveBuckets = []int64{6, -4, 1, -1, 2, 1, -4} // {6, 2, 3, 2, 4, 5, 1} @@ -1067,15 +1181,11 @@ func TestHistogramChunkAppendableGauge(t *testing.T) { require.Empty(t, nBackwardI) require.True(t, ok) - newc, recoded, _, err := hApp.AppendHistogram(nil, ts+1, h2, false) - require.NoError(t, err) - require.Nil(t, newc) - require.False(t, recoded) - require.Equal(t, GaugeType, c.(*HistogramChunk).GetCounterResetHeader()) + assertNoNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType) } { // New histogram that has a counter reset while new buckets were added. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.PositiveSpans = []histogram.Span{ {Offset: 0, Length: 3}, @@ -1093,17 +1203,13 @@ func TestHistogramChunkAppendableGauge(t *testing.T) { require.Empty(t, nBackwardI) require.True(t, ok) - newc, recoded, _, err := hApp.AppendHistogram(nil, ts+1, h2, false) - require.NoError(t, err) - require.NotNil(t, newc) - require.True(t, recoded) - require.Equal(t, GaugeType, c.(*HistogramChunk).GetCounterResetHeader()) + assertRecodedHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType) } { // New histogram that has a counter reset while new buckets were // added before the first bucket and reset on first bucket. - c, hApp, ts, h1 := setup() + c, hApp, ts, h1 := setup(eh) h2 := h1.Copy() h2.PositiveSpans = []histogram.Span{ {Offset: -3, Length: 2}, @@ -1123,11 +1229,74 @@ func TestHistogramChunkAppendableGauge(t *testing.T) { require.Empty(t, nBackwardI) require.True(t, ok) - newc, recoded, _, err := hApp.AppendHistogram(nil, ts+1, h2, false) - require.NoError(t, err) - require.NotNil(t, newc) - require.True(t, recoded) - require.Equal(t, GaugeType, c.(*HistogramChunk).GetCounterResetHeader()) + assertRecodedHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType) + } + + { // Custom buckets, no change. + c, hApp, ts, h1 := setup(cbh) + h2 := h1.Copy() + _, _, _, _, _, _, ok := hApp.appendableGauge(h2) + require.True(t, ok) + + assertNoNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType) + } + + { // Custom buckets, increase in bucket counts but no change in layout. + c, hApp, ts, h1 := setup(cbh) + h2 := h1.Copy() + h2.Count++ + h2.PositiveBuckets = []int64{6, -3, 0, -1, 2, 1, -3} + _, _, _, _, _, _, ok := hApp.appendableGauge(h2) + require.True(t, ok) + + assertNoNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType) + } + + { // Custom buckets, decrease in bucket counts but no change in layout. + c, hApp, ts, h1 := setup(cbh) + h2 := h1.Copy() + h2.Count-- + h2.PositiveBuckets = []int64{6, -3, 0, -1, 2, 1, -5} + _, _, _, _, _, _, ok := hApp.appendableGauge(h2) + require.True(t, ok) + + assertNoNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType) + } + + { // Custom buckets, change only in custom bounds. + c, hApp, ts, h1 := setup(cbh) + h2 := h1.Copy() + h2.CustomValues = []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21} + _, _, _, _, _, _, ok := hApp.appendableGauge(h2) + require.False(t, ok) + + assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType) + } + + { // Custom buckets, with more buckets. + c, hApp, ts, h1 := setup(cbh) + h2 := h1.Copy() + h2.PositiveSpans = []histogram.Span{ + {Offset: 0, Length: 3}, + {Offset: 1, Length: 1}, + {Offset: 1, Length: 4}, + {Offset: 3, Length: 3}, + } + h2.Count += 6 + h2.Sum = 30 + // Existing histogram should get values converted from the above to: + // 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between) + // so the new histogram should have new counts >= these per-bucket counts, e.g.: + h2.PositiveBuckets = []int64{7, -2, -4, 2, -2, -1, 2, 3, 0, -5, 1} // 7 5 1 3 1 0 2 5 5 0 1 (total 30) + + posInterjections, negInterjections, pBackwardI, nBackwardI, _, _, ok := hApp.appendableGauge(h2) + require.NotEmpty(t, posInterjections) + require.Empty(t, negInterjections) + require.Empty(t, pBackwardI) + require.Empty(t, nBackwardI) + require.True(t, ok) // Only new buckets came in. + + assertRecodedHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType) } } @@ -1176,4 +1345,26 @@ func TestHistogramAppendOnlyErrors(t *testing.T) { require.False(t, isRecoded) require.EqualError(t, err, "histogram counter reset") }) + t.Run("counter reset error with custom buckets", func(t *testing.T) { + c := Chunk(NewHistogramChunk()) + + // Create fresh appender and add the first histogram. + app, err := c.Appender() + require.NoError(t, err) + + h := tsdbutil.GenerateTestCustomBucketsHistogram(0) + var isRecoded bool + c, isRecoded, app, err = app.AppendHistogram(nil, 1, h, true) + require.Nil(t, c) + require.False(t, isRecoded) + require.NoError(t, err) + + // Add erroring histogram. + h2 := h.Copy() + h2.CustomValues = []float64{0, 1, 2, 3, 4, 5, 6, 7} + c, isRecoded, _, err = app.AppendHistogram(nil, 2, h2, true) + require.Nil(t, c) + require.False(t, isRecoded) + require.EqualError(t, err, "histogram counter reset") + }) } diff --git a/tsdb/tsdbutil/histogram.go b/tsdb/tsdbutil/histogram.go index bb8d49b202..3c7349cf72 100644 --- a/tsdb/tsdbutil/histogram.go +++ b/tsdb/tsdbutil/histogram.go @@ -59,6 +59,20 @@ func GenerateTestHistogram(i int) *histogram.Histogram { } } +func GenerateTestCustomBucketsHistogram(i int) *histogram.Histogram { + return &histogram.Histogram{ + Count: 5 + uint64(i*4), + Sum: 18.4 * float64(i+1), + Schema: histogram.CustomBucketsSchema, + PositiveSpans: []histogram.Span{ + {Offset: 0, Length: 2}, + {Offset: 1, Length: 2}, + }, + PositiveBuckets: []int64{int64(i + 1), 1, -1, 0}, + CustomValues: []float64{0, 1, 2, 3, 4}, + } +} + func GenerateTestGaugeHistograms(n int) (r []*histogram.Histogram) { for x := 0; x < n; x++ { i := int(math.Sin(float64(x))*100) + 100 @@ -105,6 +119,20 @@ func GenerateTestFloatHistogram(i int) *histogram.FloatHistogram { } } +func GenerateTestCustomBucketsFloatHistogram(i int) *histogram.FloatHistogram { + return &histogram.FloatHistogram{ + Count: 5 + float64(i*4), + Sum: 18.4 * float64(i+1), + Schema: histogram.CustomBucketsSchema, + PositiveSpans: []histogram.Span{ + {Offset: 0, Length: 2}, + {Offset: 1, Length: 2}, + }, + PositiveBuckets: []float64{float64(i + 1), float64(i + 2), float64(i + 1), float64(i + 1)}, + CustomValues: []float64{0, 1, 2, 3, 4}, + } +} + func GenerateTestGaugeFloatHistograms(n int) (r []*histogram.FloatHistogram) { for x := 0; x < n; x++ { i := int(math.Sin(float64(x))*100) + 100