web: Add NHCB support to federation

This simply fills the classic buckets of the histogram protobuf with
the content of the custom buckets.

Signed-off-by: beorn7 <beorn@grafana.com>

web/federate.go

@@ -190,10 +190,13 @@ Loop:
 		isHistogram := s.H != nil
 		formatType := format.FormatType()
 		if isHistogram &&
+			!s.H.UsesCustomBuckets() &&
 			formatType != expfmt.TypeProtoDelim &&
 			formatType != expfmt.TypeProtoText &&
 			formatType != expfmt.TypeProtoCompact {
-			// Can't serve the native histogram.
+			// Can't serve a native histogram with a non-protobuf format.
+			// (We can serve an NHCB, though, as it is converted to a
+			// classic histogram for federation.)
 			// TODO(codesome): Serve them when other protocols get the native histogram support.
 			continue
 		}
@@ -290,32 +293,10 @@ Loop:
 			}
 		} else {
 			lastHistogramWasGauge = s.H.CounterResetHint == histogram.GaugeType
-			protMetric.Histogram = &dto.Histogram{
+			if s.H.UsesCustomBuckets() {
-				SampleCountFloat: proto.Float64(s.H.Count),
+				protMetric.Histogram = makeClassicHistogram(s.H)
-				SampleSum:        proto.Float64(s.H.Sum),
+			} else {
-				Schema:           proto.Int32(s.H.Schema),
+				protMetric.Histogram = makeNativeHistogram(s.H)
-				ZeroThreshold:    proto.Float64(s.H.ZeroThreshold),
-				ZeroCountFloat:   proto.Float64(s.H.ZeroCount),
-				NegativeCount:    s.H.NegativeBuckets,
-				PositiveCount:    s.H.PositiveBuckets,
-			}
-			if len(s.H.PositiveSpans) > 0 {
-				protMetric.Histogram.PositiveSpan = make([]*dto.BucketSpan, len(s.H.PositiveSpans))
-				for i, sp := range s.H.PositiveSpans {
-					protMetric.Histogram.PositiveSpan[i] = &dto.BucketSpan{
-						Offset: proto.Int32(sp.Offset),
-						Length: proto.Uint32(sp.Length),
-					}
-				}
-			}
-			if len(s.H.NegativeSpans) > 0 {
-				protMetric.Histogram.NegativeSpan = make([]*dto.BucketSpan, len(s.H.NegativeSpans))
-				for i, sp := range s.H.NegativeSpans {
-					protMetric.Histogram.NegativeSpan[i] = &dto.BucketSpan{
-						Offset: proto.Int32(sp.Offset),
-						Length: proto.Uint32(sp.Length),
-					}
-				}
 			}
 		}
 		lastWasHistogram = isHistogram
@@ -329,3 +310,68 @@ Loop:
 		}
 	}
 }
+
+// makeNativeHistogram creates a dto.Histogram representing a native histogram.
+// Use only for standard exponential schemas.
+func makeNativeHistogram(h *histogram.FloatHistogram) *dto.Histogram {
+	result := &dto.Histogram{
+		SampleCountFloat: proto.Float64(h.Count),
+		SampleSum:        proto.Float64(h.Sum),
+		Schema:           proto.Int32(h.Schema),
+		ZeroThreshold:    proto.Float64(h.ZeroThreshold),
+		ZeroCountFloat:   proto.Float64(h.ZeroCount),
+		NegativeCount:    h.NegativeBuckets,
+		PositiveCount:    h.PositiveBuckets,
+	}
+	if len(h.PositiveSpans) > 0 {
+		result.PositiveSpan = make([]*dto.BucketSpan, len(h.PositiveSpans))
+		for i, sp := range h.PositiveSpans {
+			result.PositiveSpan[i] = &dto.BucketSpan{
+				Offset: proto.Int32(sp.Offset),
+				Length: proto.Uint32(sp.Length),
+			}
+		}
+	}
+	if len(h.NegativeSpans) > 0 {
+		result.NegativeSpan = make([]*dto.BucketSpan, len(h.NegativeSpans))
+		for i, sp := range h.NegativeSpans {
+			result.NegativeSpan[i] = &dto.BucketSpan{
+				Offset: proto.Int32(sp.Offset),
+				Length: proto.Uint32(sp.Length),
+			}
+		}
+	}
+	return result
+}
+
+// makeClassicHistogram creates a dto.Histogram representing a classic
+// histogram. Use only for NHCB (schema -53).
+func makeClassicHistogram(h *histogram.FloatHistogram) *dto.Histogram {
+	result := &dto.Histogram{
+		SampleCountFloat: proto.Float64(h.Count),
+		SampleSum:        proto.Float64(h.Sum),
+	}
+	result.Bucket = make([]*dto.Bucket, len(h.CustomValues))
+	var (
+		cumulativeCount float64
+		bucketIter      = h.PositiveBucketIterator()
+		bucketAvailable = bucketIter.Next()
+	)
+	for i, le := range h.CustomValues {
+		for bucketAvailable && int(bucketIter.At().Index) < i {
+			bucketAvailable = bucketIter.Next()
+		}
+		if bucketAvailable && int(bucketIter.At().Index) == i {
+			cumulativeCount += bucketIter.At().Count
+		}
+		result.Bucket[i] = &dto.Bucket{
+			UpperBound:           proto.Float64(le),
+			CumulativeCountFloat: proto.Float64(cumulativeCount),
+		}
+	}
+	// Note that we do not add the +Inf bucket explicitly. In the protobuf
+	// exposition format, it is optional. For other exposition formats, the
+	// code converting the protobuf created here into the actual exposition
+	// payload will add the +Inf bucket.
+	return result
+}

web/federate_test.go

@@ -340,8 +340,19 @@ func TestFederationWithNativeHistograms(t *testing.T) {
 		},
 		NegativeBuckets: []int64{2, 2, -2, 0},
 	}
+	nhcb := &histogram.Histogram{
+		Count:  6,
+		Sum:    1.234,
+		Schema: -53,
+		PositiveSpans: []histogram.Span{
+			{Offset: 0, Length: 2},
+			{Offset: 2, Length: 1},
+		},
+		PositiveBuckets: []int64{3, -1, -1},
+		CustomValues:    []float64{0.1, 0.2, 0.5, 1, 2},
+	}
 	app := db.Appender(context.Background())
-	for i := range 6 {
+	for i := range 7 {
 		l := labels.FromStrings("__name__", "test_metric", "foo", strconv.Itoa(i))
 		expL := labels.FromStrings("__name__", "test_metric", "instance", "", "foo", strconv.Itoa(i))
 		var err error
@@ -360,6 +371,56 @@ func TestFederationWithNativeHistograms(t *testing.T) {
 				H:      histWithoutZeroBucket.ToFloat(nil),
 				Metric: expL,
 			})
+		case 6:
+			_, err = app.AppendHistogram(0, l, 100*60*1000, nhcb.Copy(), nil)
+			expL = labels.FromStrings("__name__", "test_metric_count", "instance", "", "foo", strconv.Itoa(i))
+			expVec = append(expVec, promql.Sample{
+				T:      100 * 60 * 1000,
+				F:      6,
+				Metric: expL,
+			})
+			expL = labels.FromStrings("__name__", "test_metric_sum", "instance", "", "foo", strconv.Itoa(i))
+			expVec = append(expVec, promql.Sample{
+				T:      100 * 60 * 1000,
+				F:      1.234,
+				Metric: expL,
+			})
+			expL = labels.FromStrings("__name__", "test_metric_bucket", "instance", "", "foo", strconv.Itoa(i), "le", "0.1")
+			expVec = append(expVec, promql.Sample{
+				T:      100 * 60 * 1000,
+				F:      3,
+				Metric: expL,
+			})
+			expL = labels.FromStrings("__name__", "test_metric_bucket", "instance", "", "foo", strconv.Itoa(i), "le", "0.2")
+			expVec = append(expVec, promql.Sample{
+				T:      100 * 60 * 1000,
+				F:      5,
+				Metric: expL,
+			})
+			expL = labels.FromStrings("__name__", "test_metric_bucket", "instance", "", "foo", strconv.Itoa(i), "le", "0.5")
+			expVec = append(expVec, promql.Sample{
+				T:      100 * 60 * 1000,
+				F:      5,
+				Metric: expL,
+			})
+			expL = labels.FromStrings("__name__", "test_metric_bucket", "instance", "", "foo", strconv.Itoa(i), "le", "1.0")
+			expVec = append(expVec, promql.Sample{
+				T:      100 * 60 * 1000,
+				F:      5,
+				Metric: expL,
+			})
+			expL = labels.FromStrings("__name__", "test_metric_bucket", "instance", "", "foo", strconv.Itoa(i), "le", "2.0")
+			expVec = append(expVec, promql.Sample{
+				T:      100 * 60 * 1000,
+				F:      6,
+				Metric: expL,
+			})
+			expL = labels.FromStrings("__name__", "test_metric_bucket", "instance", "", "foo", strconv.Itoa(i), "le", "+Inf")
+			expVec = append(expVec, promql.Sample{
+				T:      100 * 60 * 1000,
+				F:      6,
+				Metric: expL,
+			})
 		default:
 			hist.ZeroCount++
 			hist.Count++