From 3e96d902e0c6b388a938c78af7362d5bfe49e39b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gy=C3=B6rgy=20Krajcsovits?= Date: Thu, 18 Sep 2025 13:12:18 +0200 Subject: [PATCH] feat(remote): reduce resolution of native histograms on remote read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a sample read through remote read has too high resolution, reduce it to the maximum allowed. This is a slow path, but we only expect it to happen if the server side is newer version that allows higher resolution. Signed-off-by: György Krajcsovits --- storage/remote/codec.go | 20 +++++++++++++++++--- storage/remote/codec_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/storage/remote/codec.go b/storage/remote/codec.go index 2042e12199..2031d6101d 100644 --- a/storage/remote/codec.go +++ b/storage/remote/codec.go @@ -472,7 +472,7 @@ func (c *concreteSeriesIterator) Seek(t int64) chunkenc.ValueType { } func validateHistogramSchema(h *prompb.Histogram) error { - if histogram.IsValidSchema(h.Schema) { + if histogram.IsAcceptibleSchema(h.Schema) { return nil } return fmt.Errorf("invalid histogram schema %d", h.Schema) @@ -500,14 +500,28 @@ func (c *concreteSeriesIterator) AtHistogram(*histogram.Histogram) (int64, *hist panic("iterator is not on an integer histogram sample") } h := c.series.histograms[c.histogramsCur] - return h.Timestamp, h.ToIntHistogram() + mh := h.ToIntHistogram() + if mh.Schema > histogram.ExponentialSchemaMax && mh.Schema <= histogram.ExponentialSchemaMaxReserved { + // This is a very slow path, but it should only happen if the + // sample is from a newer Prometheus version that supports higher + // resolution. + mh.ReduceResolution(histogram.ExponentialSchemaMax) + } + return h.Timestamp, mh } // AtFloatHistogram implements chunkenc.Iterator. func (c *concreteSeriesIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) { if c.curValType == chunkenc.ValHistogram || c.curValType == chunkenc.ValFloatHistogram { fh := c.series.histograms[c.histogramsCur] - return fh.Timestamp, fh.ToFloatHistogram() // integer will be auto-converted. + mfh := fh.ToFloatHistogram() // integer will be auto-converted. + if mfh.Schema > histogram.ExponentialSchemaMax && mfh.Schema <= histogram.ExponentialSchemaMaxReserved { + // This is a very slow path, but it should only happen if the + // sample is from a newer Prometheus version that supports higher + // resolution. + mfh.ReduceResolution(histogram.ExponentialSchemaMax) + } + return fh.Timestamp, mfh } panic("iterator is not on a histogram sample") } diff --git a/storage/remote/codec_test.go b/storage/remote/codec_test.go index 396acd0a5f..da96519b70 100644 --- a/storage/remote/codec_test.go +++ b/storage/remote/codec_test.go @@ -592,6 +592,34 @@ func TestConcreteSeriesIterator_InvalidHistogramSamples(t *testing.T) { } } +func TestConcreteSeriesIterator_ReducesHighResolutionHistograms(t *testing.T) { + for _, schema := range []int32{9, 52} { + t.Run(fmt.Sprintf("schema=%d", schema), func(t *testing.T) { + h := testHistogram.Copy() + h.Schema = schema + fh := h.ToFloat(nil) + series := &concreteSeries{ + labels: labels.FromStrings("foo", "bar"), + histograms: []prompb.Histogram{ + prompb.FromIntHistogram(1, h), + prompb.FromFloatHistogram(2, fh), + }, + } + it := series.Iterator(nil) + require.Equal(t, chunkenc.ValHistogram, it.Next()) + _, gotH := it.AtHistogram(nil) + require.Equal(t, histogram.ExponentialSchemaMax, gotH.Schema) + _, gotFH := it.AtFloatHistogram(nil) + require.Equal(t, histogram.ExponentialSchemaMax, gotFH.Schema) + require.Equal(t, chunkenc.ValFloatHistogram, it.Next()) + _, gotFH = it.AtFloatHistogram(nil) + require.Equal(t, histogram.ExponentialSchemaMax, gotFH.Schema) + require.Equal(t, chunkenc.ValNone, it.Next()) + require.NoError(t, it.Err()) + }) + } +} + func TestFromQueryResultWithDuplicates(t *testing.T) { ts1 := prompb.TimeSeries{ Labels: []prompb.Label{