diff --git a/model/histogram/float_histogram.go b/model/histogram/float_histogram.go index 91fcac1cfb..0acf9cb28f 100644 --- a/model/histogram/float_histogram.go +++ b/model/histogram/float_histogram.go @@ -484,7 +484,7 @@ func (h *FloatHistogram) Sub(other *FloatHistogram) (res *FloatHistogram, counte // supposed to be used according to the schema. func (h *FloatHistogram) Equals(h2 *FloatHistogram) bool { if h2 == nil { - return false + return h == nil } if h.Schema != h2.Schema || diff --git a/model/histogram/histogram.go b/model/histogram/histogram.go index 5fc68ef9d0..aa9f696be6 100644 --- a/model/histogram/histogram.go +++ b/model/histogram/histogram.go @@ -247,7 +247,7 @@ func (h *Histogram) CumulativeBucketIterator() BucketIterator[uint64] { // supposed to be used according to the schema. func (h *Histogram) Equals(h2 *Histogram) bool { if h2 == nil { - return false + return h == nil } if h.Schema != h2.Schema || h.Count != h2.Count || diff --git a/promql/promqltest/test.go b/promql/promqltest/test.go index b16433c14e..d331b7adf3 100644 --- a/promql/promqltest/test.go +++ b/promql/promqltest/test.go @@ -648,12 +648,12 @@ func (cmd *loadCmd) set(m labels.Labels, vals ...parser.SequenceValue) { } // append the defined time series to the storage. -func (cmd *loadCmd) append(a storage.Appender) error { +func (cmd *loadCmd) append(a storage.AppenderV2) error { for h, smpls := range cmd.defs { - m := cmd.metrics[h] + ls := cmd.metrics[h] for _, s := range smpls { - if err := appendSample(a, s, m); err != nil { + if _, err := a.Append(0, ls, 0, s.T, s.F, nil, s.H, storage.AOptions{}); err != nil { return err } } @@ -699,7 +699,7 @@ func processClassicHistogramSeries(m labels.Labels, name string, histogramMap ma // If classic histograms are defined, convert them into native histograms with custom // bounds and append the defined time series to the storage. -func (cmd *loadCmd) appendCustomHistogram(a storage.Appender) error { +func (cmd *loadCmd) appendCustomHistogram(a storage.AppenderV2) error { histogramMap := map[uint64]tempHistogramWrapper{} // Go through all the time series to collate classic histogram data @@ -754,7 +754,7 @@ func (cmd *loadCmd) appendCustomHistogram(a storage.Appender) error { } sort.Slice(samples, func(i, j int) bool { return samples[i].T < samples[j].T }) for _, s := range samples { - if err := appendSample(a, s, histogramWrapper.metric); err != nil { + if _, err := a.Append(0, histogramWrapper.metric, 0, s.T, s.F, nil, s.H, storage.AOptions{}); err != nil { return err } } @@ -762,19 +762,6 @@ func (cmd *loadCmd) appendCustomHistogram(a storage.Appender) error { return nil } -func appendSample(a storage.Appender, s promql.Sample, m labels.Labels) error { - if s.H != nil { - if _, err := a.AppendHistogram(0, m, s.T, nil, s.H); err != nil { - return err - } - } else { - if _, err := a.Append(0, m, s.T, s.F); err != nil { - return err - } - } - return nil -} - // evalCmd is a command that evaluates an expression for the given time (range) // and expects a specific result. type evalCmd struct { @@ -1386,7 +1373,7 @@ func (t *test) exec(tc testCommand, engine promql.QueryEngine) error { t.clear() case *loadCmd: - app := t.storage.Appender(t.context) + app := t.storage.AppenderV2(t.context) if err := cmd.append(app); err != nil { app.Rollback() return err @@ -1699,16 +1686,16 @@ func (ll *LazyLoader) clear() error { // appendTill appends the defined time series to the storage till the given timestamp (in milliseconds). func (ll *LazyLoader) appendTill(ts int64) error { - app := ll.storage.Appender(ll.Context()) + app := ll.storage.AppenderV2(ll.Context()) for h, smpls := range ll.loadCmd.defs { - m := ll.loadCmd.metrics[h] + ls := ll.loadCmd.metrics[h] for i, s := range smpls { if s.T > ts { // Removing the already added samples. ll.loadCmd.defs[h] = smpls[i:] break } - if err := appendSample(app, s, m); err != nil { + if _, err := app.Append(0, ls, 0, s.T, s.F, nil, s.H, storage.AOptions{}); err != nil { return err } if i == len(smpls)-1 { diff --git a/scrape/helpers_test.go b/scrape/helpers_test.go index ff7a7bf65a..614d066f02 100644 --- a/scrape/helpers_test.go +++ b/scrape/helpers_test.go @@ -17,242 +17,32 @@ import ( "bytes" "context" "encoding/binary" - "fmt" - "math" - "strings" - "sync" "testing" "github.com/gogo/protobuf/proto" dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/require" - "github.com/prometheus/prometheus/model/exemplar" "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/model/metadata" "github.com/prometheus/prometheus/storage" ) type nopAppendable struct{} -func (nopAppendable) Appender(context.Context) storage.Appender { +func (nopAppendable) AppenderV2(context.Context) storage.AppenderV2 { return nopAppender{} } type nopAppender struct{} -func (nopAppender) SetOptions(*storage.AppendOptions) {} - -func (nopAppender) Append(storage.SeriesRef, labels.Labels, int64, float64) (storage.SeriesRef, error) { +func (nopAppender) Append(ref storage.SeriesRef, ls labels.Labels, st, t int64, v float64, h *histogram.Histogram, fh *histogram.FloatHistogram, opts storage.AOptions) (storage.SeriesRef, error) { return 1, nil } -func (nopAppender) AppendExemplar(storage.SeriesRef, labels.Labels, exemplar.Exemplar) (storage.SeriesRef, error) { - return 2, nil -} - -func (nopAppender) AppendHistogram(storage.SeriesRef, labels.Labels, int64, *histogram.Histogram, *histogram.FloatHistogram) (storage.SeriesRef, error) { - return 3, nil -} - -func (nopAppender) AppendHistogramSTZeroSample(storage.SeriesRef, labels.Labels, int64, int64, *histogram.Histogram, *histogram.FloatHistogram) (storage.SeriesRef, error) { - return 0, nil -} - -func (nopAppender) UpdateMetadata(storage.SeriesRef, labels.Labels, metadata.Metadata) (storage.SeriesRef, error) { - return 4, nil -} - -func (nopAppender) AppendSTZeroSample(storage.SeriesRef, labels.Labels, int64, int64) (storage.SeriesRef, error) { - return 5, nil -} - func (nopAppender) Commit() error { return nil } func (nopAppender) Rollback() error { return nil } -type floatSample struct { - metric labels.Labels - t int64 - f float64 -} - -func equalFloatSamples(a, b floatSample) bool { - // Compare Float64bits so NaN values which are exactly the same will compare equal. - return labels.Equal(a.metric, b.metric) && a.t == b.t && math.Float64bits(a.f) == math.Float64bits(b.f) -} - -type histogramSample struct { - metric labels.Labels - t int64 - h *histogram.Histogram - fh *histogram.FloatHistogram -} - -type metadataEntry struct { - m metadata.Metadata - metric labels.Labels -} - -func metadataEntryEqual(a, b metadataEntry) bool { - if !labels.Equal(a.metric, b.metric) { - return false - } - if a.m.Type != b.m.Type { - return false - } - if a.m.Unit != b.m.Unit { - return false - } - if a.m.Help != b.m.Help { - return false - } - return true -} - -type collectResultAppendable struct { - *collectResultAppender -} - -func (a *collectResultAppendable) Appender(context.Context) storage.Appender { - return a -} - -// collectResultAppender records all samples that were added through the appender. -// It can be used as its zero value or be backed by another appender it writes samples through. -type collectResultAppender struct { - mtx sync.Mutex - - next storage.Appender - resultFloats []floatSample - pendingFloats []floatSample - rolledbackFloats []floatSample - resultHistograms []histogramSample - pendingHistograms []histogramSample - rolledbackHistograms []histogramSample - resultExemplars []exemplar.Exemplar - pendingExemplars []exemplar.Exemplar - resultMetadata []metadataEntry - pendingMetadata []metadataEntry -} - -func (*collectResultAppender) SetOptions(*storage.AppendOptions) {} - -func (a *collectResultAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64, v float64) (storage.SeriesRef, error) { - a.mtx.Lock() - defer a.mtx.Unlock() - a.pendingFloats = append(a.pendingFloats, floatSample{ - metric: lset, - t: t, - f: v, - }) - - if a.next == nil { - if ref == 0 { - // Use labels hash as a stand-in for unique series reference, to avoid having to track all series. - ref = storage.SeriesRef(lset.Hash()) - } - return ref, nil - } - - ref, err := a.next.Append(ref, lset, t, v) - if err != nil { - return 0, err - } - return ref, nil -} - -func (a *collectResultAppender) AppendExemplar(ref storage.SeriesRef, l labels.Labels, e exemplar.Exemplar) (storage.SeriesRef, error) { - a.mtx.Lock() - defer a.mtx.Unlock() - a.pendingExemplars = append(a.pendingExemplars, e) - if a.next == nil { - return 0, nil - } - - return a.next.AppendExemplar(ref, l, e) -} - -func (a *collectResultAppender) AppendHistogram(ref storage.SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { - a.mtx.Lock() - defer a.mtx.Unlock() - a.pendingHistograms = append(a.pendingHistograms, histogramSample{h: h, fh: fh, t: t, metric: l}) - if a.next == nil { - return 0, nil - } - - return a.next.AppendHistogram(ref, l, t, h, fh) -} - -func (a *collectResultAppender) AppendHistogramSTZeroSample(ref storage.SeriesRef, l labels.Labels, _, st int64, h *histogram.Histogram, _ *histogram.FloatHistogram) (storage.SeriesRef, error) { - if h != nil { - return a.AppendHistogram(ref, l, st, &histogram.Histogram{}, nil) - } - return a.AppendHistogram(ref, l, st, nil, &histogram.FloatHistogram{}) -} - -func (a *collectResultAppender) UpdateMetadata(ref storage.SeriesRef, l labels.Labels, m metadata.Metadata) (storage.SeriesRef, error) { - a.mtx.Lock() - defer a.mtx.Unlock() - a.pendingMetadata = append(a.pendingMetadata, metadataEntry{metric: l, m: m}) - if a.next == nil { - if ref == 0 { - ref = storage.SeriesRef(l.Hash()) - } - return ref, nil - } - - return a.next.UpdateMetadata(ref, l, m) -} - -func (a *collectResultAppender) AppendSTZeroSample(ref storage.SeriesRef, l labels.Labels, _, st int64) (storage.SeriesRef, error) { - return a.Append(ref, l, st, 0.0) -} - -func (a *collectResultAppender) Commit() error { - a.mtx.Lock() - defer a.mtx.Unlock() - a.resultFloats = append(a.resultFloats, a.pendingFloats...) - a.resultExemplars = append(a.resultExemplars, a.pendingExemplars...) - a.resultHistograms = append(a.resultHistograms, a.pendingHistograms...) - a.resultMetadata = append(a.resultMetadata, a.pendingMetadata...) - a.pendingFloats = nil - a.pendingExemplars = nil - a.pendingHistograms = nil - a.pendingMetadata = nil - if a.next == nil { - return nil - } - return a.next.Commit() -} - -func (a *collectResultAppender) Rollback() error { - a.mtx.Lock() - defer a.mtx.Unlock() - a.rolledbackFloats = a.pendingFloats - a.rolledbackHistograms = a.pendingHistograms - a.pendingFloats = nil - a.pendingHistograms = nil - if a.next == nil { - return nil - } - return a.next.Rollback() -} - -func (a *collectResultAppender) String() string { - var sb strings.Builder - for _, s := range a.resultFloats { - sb.WriteString(fmt.Sprintf("committed: %s %f %d\n", s.metric, s.f, s.t)) - } - for _, s := range a.pendingFloats { - sb.WriteString(fmt.Sprintf("pending: %s %f %d\n", s.metric, s.f, s.t)) - } - for _, s := range a.rolledbackFloats { - sb.WriteString(fmt.Sprintf("rolledback: %s %f %d\n", s.metric, s.f, s.t)) - } - return sb.String() -} - // protoMarshalDelimited marshals a MetricFamily into a delimited // Prometheus proto exposition format bytes (known as `encoding=delimited`) // diff --git a/scrape/manager.go b/scrape/manager.go index c63d7d0eae..5e90596ef7 100644 --- a/scrape/manager.go +++ b/scrape/manager.go @@ -38,8 +38,10 @@ import ( "github.com/prometheus/prometheus/util/pool" ) -// NewManager is the Manager constructor. -func NewManager(o *Options, logger *slog.Logger, newScrapeFailureLogger func(string) (*logging.JSONFileLogger, error), app storage.Appendable, registerer prometheus.Registerer) (*Manager, error) { +// NewManager is the Manager constructor using deprecated Appendable. +// +// Deprecated: Use NewManagerV2 instead. NewManager will be removed (or replaced with NewManagerV2) soon (ETA: Q2 2026). +func NewManager(o *Options, logger *slog.Logger, newScrapeFailureLogger func(string) (*logging.JSONFileLogger, error), appendableV1 storage.Appendable, registerer prometheus.Registerer) (*Manager, error) { if o == nil { o = &Options{} } @@ -53,7 +55,39 @@ func NewManager(o *Options, logger *slog.Logger, newScrapeFailureLogger func(str } m := &Manager{ - append: app, + appendableV1: appendableV1, + opts: o, + logger: logger, + newScrapeFailureLogger: newScrapeFailureLogger, + scrapeConfigs: make(map[string]*config.ScrapeConfig), + scrapePools: make(map[string]*scrapePool), + graceShut: make(chan struct{}), + triggerReload: make(chan struct{}, 1), + metrics: sm, + buffers: pool.New(1e3, 100e6, 3, func(sz int) any { return make([]byte, 0, sz) }), + } + + m.metrics.setTargetMetadataCacheGatherer(m) + + return m, nil +} + +// NewManagerWithAppendableV2 is the Manager constructor using AppendableV2. +func NewManagerWithAppendableV2(o *Options, logger *slog.Logger, newScrapeFailureLogger func(string) (*logging.JSONFileLogger, error), appendableV2 storage.AppendableV2, registerer prometheus.Registerer) (*Manager, error) { + if o == nil { + o = &Options{} + } + if logger == nil { + logger = promslog.NewNopLogger() + } + + sm, err := newScrapeMetrics(registerer) + if err != nil { + return nil, fmt.Errorf("failed to create scrape manager due to error: %w", err) + } + + m := &Manager{ + appendableV2: appendableV2, opts: o, logger: logger, newScrapeFailureLogger: newScrapeFailureLogger, @@ -75,19 +109,22 @@ type Options struct { ExtraMetrics bool // Option used by downstream scraper users like OpenTelemetry Collector // to help lookup metric metadata. Should be false for Prometheus. + // TODO(bwplotka): Remove once appender v1 flow is removed, collector can use AppenderV2 + // which is capable of passing metadata on every Append. PassMetadataInContext bool // Option to enable appending of scraped Metadata to the TSDB/other appenders. Individual appenders // can decide what to do with metadata, but for practical purposes this flag exists so that metadata // can be written to the WAL and thus read for remote write. - // TODO: implement some form of metadata storage AppendMetadata bool // Option to increase the interval used by scrape manager to throttle target groups updates. DiscoveryReloadInterval model.Duration + // Option to enable the ingestion of the created timestamp as a synthetic zero sample. // See: https://github.com/prometheus/proposals/blob/main/proposals/2023-06-13_created-timestamp.md + // TODO(bwplotka): Remove once appender v1 flow is removed. EnableStartTimestampZeroIngestion bool - // EnableTypeAndUnitLabels + // EnableTypeAndUnitLabels represents type-and-unit-labels feature flag. EnableTypeAndUnitLabels bool // Optional HTTP client options to use when scraping. @@ -100,9 +137,12 @@ type Options struct { // Manager maintains a set of scrape pools and manages start/stop cycles // when receiving new target groups from the discovery manager. type Manager struct { - opts *Options - logger *slog.Logger - append storage.Appendable + opts *Options + logger *slog.Logger + + appendableV1 storage.Appendable + appendableV2 storage.AppendableV2 + graceShut chan struct{} offsetSeed uint64 // Global offsetSeed seed is used to spread scrape workload across HA setup. @@ -183,7 +223,7 @@ func (m *Manager) reload() { continue } m.metrics.targetScrapePools.Inc() - sp, err := newScrapePool(scrapeConfig, m.append, m.offsetSeed, m.logger.With("scrape_pool", setName), m.buffers, m.opts, m.metrics) + sp, err := newScrapePool(scrapeConfig, m.appendableV1, m.appendableV2, m.offsetSeed, m.logger.With("scrape_pool", setName), m.buffers, m.opts, m.metrics) if err != nil { m.metrics.targetScrapePoolsFailed.Inc() m.logger.Error("error creating new scrape pool", "err", err, "scrape_pool", setName) diff --git a/scrape/manager_test.go b/scrape/manager_test.go index 1ec4875d19..5e14ee79f4 100644 --- a/scrape/manager_test.go +++ b/scrape/manager_test.go @@ -30,25 +30,24 @@ import ( "testing" "time" - "github.com/gogo/protobuf/proto" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" "github.com/prometheus/common/model" "github.com/prometheus/common/promslog" + "github.com/prometheus/prometheus/util/teststorage" "github.com/stretchr/testify/require" "go.yaml.in/yaml/v2" + "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/discovery" _ "github.com/prometheus/prometheus/discovery/file" "github.com/prometheus/prometheus/discovery/targetgroup" - "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/relabel" "github.com/prometheus/prometheus/model/timestamp" - "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/tsdbutil" "github.com/prometheus/prometheus/util/runutil" "github.com/prometheus/prometheus/util/testutil" @@ -528,7 +527,7 @@ scrape_configs: return noopLoop() } sp := &scrapePool{ - appendable: &nopAppendable{}, + appendableV2: &nopAppendable{}, activeTargets: map[uint64]*Target{ 1: {}, }, @@ -692,7 +691,7 @@ scrape_configs: _, cancel := context.WithCancel(context.Background()) defer cancel() sp := &scrapePool{ - appendable: &nopAppendable{}, + appendableV2: &nopAppendable{}, activeTargets: map[uint64]*Target{}, loops: map[uint64]loop{ 1: noopLoop(), @@ -777,11 +776,10 @@ func TestManagerSTZeroIngestion(t *testing.T) { // TODO(bwplotka): Add more types than just counter? encoded := prepareTestEncodedCounter(t, testFormat, expectedMetricName, expectedSampleValue, sampleTs, stTs) - app := &collectResultAppender{} - discoveryManager, scrapeManager := runManagers(t, ctx, &Options{ + discoveryManager, scrapeManager, appTest := runManagers(t, ctx, &Options{ EnableStartTimestampZeroIngestion: testSTZeroIngest, skipOffsetting: true, - }, &collectResultAppendable{app}) + }) defer scrapeManager.Stop() server := setupTestServer(t, config.ScrapeProtocolsHeaders[testFormat], encoded) @@ -806,11 +804,8 @@ scrape_configs: ctx, cancel = context.WithTimeout(ctx, 1*time.Minute) defer cancel() require.NoError(t, runutil.Retry(100*time.Millisecond, ctx.Done(), func() error { - app.mtx.Lock() - defer app.mtx.Unlock() - // Check if scrape happened and grab the relevant samples. - if len(app.resultFloats) > 0 { + if appTest.ResultSamplesGreaterThan(0) { return nil } return errors.New("expected some float samples, got none") @@ -818,22 +813,22 @@ scrape_configs: // Verify results. // Verify what we got vs expectations around ST injection. - samples := findSamplesForMetric(app.resultFloats, expectedMetricName) + samples := findSamplesForMetric(appTest.ResultSamples, expectedMetricName) if testWithST && testSTZeroIngest { require.Len(t, samples, 2) - require.Equal(t, 0.0, samples[0].f) - require.Equal(t, timestamp.FromTime(stTs), samples[0].t) - require.Equal(t, expectedSampleValue, samples[1].f) - require.Equal(t, timestamp.FromTime(sampleTs), samples[1].t) + require.Equal(t, 0.0, samples[0].V) + require.Equal(t, timestamp.FromTime(stTs), samples[0].T) + require.Equal(t, expectedSampleValue, samples[1].V) + require.Equal(t, timestamp.FromTime(sampleTs), samples[1].T) } else { require.Len(t, samples, 1) - require.Equal(t, expectedSampleValue, samples[0].f) - require.Equal(t, timestamp.FromTime(sampleTs), samples[0].t) + require.Equal(t, expectedSampleValue, samples[0].V) + require.Equal(t, timestamp.FromTime(sampleTs), samples[0].T) } // Verify what we got vs expectations around additional _created series for OM text. // enableSTZeroInjection also kills that _created line. - createdSeriesSamples := findSamplesForMetric(app.resultFloats, expectedCreatedMetricName) + createdSeriesSamples := findSamplesForMetric(appTest.ResultSamples, expectedCreatedMetricName) if testFormat == config.OpenMetricsText1_0_0 && testWithST && !testSTZeroIngest { // For OM Text, when counter has ST, and feature flag disabled we should see _created lines. require.Len(t, createdSeriesSamples, 1) @@ -841,7 +836,7 @@ scrape_configs: // We don't check the st timestamp as explicit ts was not implemented in expfmt.Encoder, // but exists in OM https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#:~:text=An%20example%20with%20a%20Metric%20with%20no%20labels%2C%20and%20a%20MetricPoint%20with%20a%20timestamp%20and%20a%20created // We can implement this, but we want to potentially get rid of OM 1.0 ST lines - require.Equal(t, float64(timestamppb.New(stTs).AsTime().UnixNano())/1e9, createdSeriesSamples[0].f) + require.Equal(t, float64(timestamppb.New(stTs).AsTime().UnixNano())/1e9, createdSeriesSamples[0].V) } else { require.Empty(t, createdSeriesSamples) } @@ -885,9 +880,9 @@ func prepareTestEncodedCounter(t *testing.T, format config.ScrapeProtocol, mName } } -func findSamplesForMetric(floats []floatSample, metricName string) (ret []floatSample) { - for _, f := range floats { - if f.metric.Get(model.MetricNameLabel) == metricName { +func findSamplesForMetric(s []sample, metricName string) (ret []sample) { + for _, f := range s { + if f.L.Get(model.MetricNameLabel) == metricName { ret = append(ret, f) } } @@ -923,136 +918,6 @@ func generateTestHistogram(i int) *dto.Histogram { return h } -func TestManagerSTZeroIngestionHistogram(t *testing.T) { - t.Parallel() - const mName = "expected_histogram" - - for _, tc := range []struct { - name string - inputHistSample *dto.Histogram - enableSTZeroIngestion bool - }{ - { - name: "disabled with ST on histogram", - inputHistSample: func() *dto.Histogram { - h := generateTestHistogram(0) - h.CreatedTimestamp = timestamppb.Now() - return h - }(), - enableSTZeroIngestion: false, - }, - { - name: "enabled with ST on histogram", - inputHistSample: func() *dto.Histogram { - h := generateTestHistogram(0) - h.CreatedTimestamp = timestamppb.Now() - return h - }(), - enableSTZeroIngestion: true, - }, - { - name: "enabled without ST on histogram", - inputHistSample: func() *dto.Histogram { - h := generateTestHistogram(0) - return h - }(), - enableSTZeroIngestion: true, - }, - } { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - app := &collectResultAppender{} - discoveryManager, scrapeManager := runManagers(t, ctx, &Options{ - EnableStartTimestampZeroIngestion: tc.enableSTZeroIngestion, - skipOffsetting: true, - }, &collectResultAppendable{app}) - defer scrapeManager.Stop() - - once := sync.Once{} - // Start fake HTTP target to that allow one scrape only. - server := httptest.NewServer( - http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - fail := true - once.Do(func() { - fail = false - w.Header().Set("Content-Type", `application/vnd.google.protobuf; proto=io.prometheus.client.MetricFamily; encoding=delimited`) - - ctrType := dto.MetricType_HISTOGRAM - w.Write(protoMarshalDelimited(t, &dto.MetricFamily{ - Name: proto.String(mName), - Type: &ctrType, - Metric: []*dto.Metric{{Histogram: tc.inputHistSample}}, - })) - }) - - if fail { - w.WriteHeader(http.StatusInternalServerError) - } - }), - ) - defer server.Close() - - serverURL, err := url.Parse(server.URL) - require.NoError(t, err) - - testConfig := fmt.Sprintf(` -global: - # Disable regular scrapes. - scrape_interval: 9999m - scrape_timeout: 5s - -scrape_configs: -- job_name: test - scrape_native_histograms: true - static_configs: - - targets: ['%s'] -`, serverURL.Host) - applyConfig(t, testConfig, scrapeManager, discoveryManager) - - var got []histogramSample - - // Wait for one scrape. - ctx, cancel = context.WithTimeout(ctx, 1*time.Minute) - defer cancel() - require.NoError(t, runutil.Retry(100*time.Millisecond, ctx.Done(), func() error { - app.mtx.Lock() - defer app.mtx.Unlock() - - // Check if scrape happened and grab the relevant histograms, they have to be there - or it's a bug - // and it's not worth waiting. - for _, h := range app.resultHistograms { - if h.metric.Get(model.MetricNameLabel) == mName { - got = append(got, h) - } - } - if len(app.resultHistograms) > 0 { - return nil - } - return errors.New("expected some histogram samples, got none") - }), "after 1 minute") - - // Check for zero samples, assuming we only injected always one histogram sample. - // Did it contain ST to inject? If yes, was ST zero enabled? - if tc.inputHistSample.CreatedTimestamp.IsValid() && tc.enableSTZeroIngestion { - require.Len(t, got, 2) - // Zero sample. - require.Equal(t, histogram.Histogram{}, *got[0].h) - // Quick soft check to make sure it's the same sample or at least not zero. - require.Equal(t, tc.inputHistSample.GetSampleSum(), got[1].h.Sum) - return - } - - // Expect only one, valid sample. - require.Len(t, got, 1) - // Quick soft check to make sure it's the same sample or at least not zero. - require.Equal(t, tc.inputHistSample.GetSampleSum(), got[0].h.Sum) - }) - } -} - func TestUnregisterMetrics(t *testing.T) { reg := prometheus.NewRegistry() // Check that all metrics can be unregistered, allowing a second manager to be created. @@ -1066,115 +931,6 @@ func TestUnregisterMetrics(t *testing.T) { } } -// TestNHCBAndSTZeroIngestion verifies that both ConvertClassicHistogramsToNHCBEnabled -// and EnableStartTimestampZeroIngestion can be used simultaneously without errors. -// This test addresses issue #17216 by ensuring the previously blocking check has been removed. -// The test verifies that the presence of exemplars in the input does not cause errors, -// although exemplars are not preserved during NHCB conversion (as documented below). -func TestNHCBAndSTZeroIngestion(t *testing.T) { - t.Parallel() - - const ( - mName = "test_histogram" - // The expected sum of the histogram, as defined by the test's OpenMetrics exposition data. - // This value (45.5) is the sum reported in the test_histogram_sum metric below. - expectedHistogramSum = 45.5 - ) - - ctx := t.Context() - - app := &collectResultAppender{} - discoveryManager, scrapeManager := runManagers(t, ctx, &Options{ - EnableStartTimestampZeroIngestion: true, - skipOffsetting: true, - }, &collectResultAppendable{app}) - defer scrapeManager.Stop() - - once := sync.Once{} - server := httptest.NewServer( - http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - fail := true - once.Do(func() { - fail = false - w.Header().Set("Content-Type", `application/openmetrics-text`) - - // Expose a histogram with created timestamp and exemplars to verify no parsing errors occur. - fmt.Fprint(w, `# HELP test_histogram A histogram with created timestamp and exemplars -# TYPE test_histogram histogram -test_histogram_bucket{le="0.0"} 1 -test_histogram_bucket{le="1.0"} 10 # {trace_id="trace-1"} 0.5 123456789 -test_histogram_bucket{le="2.0"} 20 # {trace_id="trace-2"} 1.5 123456780 -test_histogram_bucket{le="+Inf"} 30 # {trace_id="trace-3"} 2.5 -test_histogram_count 30 -test_histogram_sum 45.5 -test_histogram_created 1520430001 -# EOF -`) - }) - - if fail { - w.WriteHeader(http.StatusInternalServerError) - } - }), - ) - defer server.Close() - - serverURL, err := url.Parse(server.URL) - require.NoError(t, err) - - // Configuration with both convert_classic_histograms_to_nhcb enabled and ST zero ingestion enabled. - testConfig := fmt.Sprintf(` -global: - # Use a very long scrape_interval to prevent automatic scraping during the test. - scrape_interval: 9999m - scrape_timeout: 5s - -scrape_configs: -- job_name: test - convert_classic_histograms_to_nhcb: true - static_configs: - - targets: ['%s'] -`, serverURL.Host) - - applyConfig(t, testConfig, scrapeManager, discoveryManager) - - // Verify that the scrape pool was created (proves the blocking check was removed). - require.Eventually(t, func() bool { - scrapeManager.mtxScrape.Lock() - defer scrapeManager.mtxScrape.Unlock() - _, exists := scrapeManager.scrapePools["test"] - return exists - }, 5*time.Second, 100*time.Millisecond, "scrape pool should be created for job 'test'") - - // Helper function to get matching histograms to avoid race conditions. - getMatchingHistograms := func() []histogramSample { - app.mtx.Lock() - defer app.mtx.Unlock() - - var got []histogramSample - for _, h := range app.resultHistograms { - if h.metric.Get(model.MetricNameLabel) == mName { - got = append(got, h) - } - } - return got - } - - require.Eventually(t, func() bool { - return len(getMatchingHistograms()) > 0 - }, 1*time.Minute, 100*time.Millisecond, "expected histogram samples, got none") - - // Verify that samples were ingested (proving both features work together). - got := getMatchingHistograms() - - // With ST zero ingestion enabled and a created timestamp present, we expect 2 samples: - // one zero sample and one actual sample. - require.Len(t, got, 2, "expected 2 histogram samples (zero sample + actual sample)") - require.Equal(t, histogram.Histogram{}, *got[0].h, "first sample should be zero sample") - require.InDelta(t, expectedHistogramSum, got[1].h.Sum, 1e-9, "second sample should retain the expected sum") - require.Len(t, app.resultExemplars, 2, "expected 2 exemplars from histogram buckets") -} - func applyConfig( t *testing.T, config string, @@ -1195,16 +951,15 @@ func applyConfig( require.NoError(t, discoveryManager.ApplyConfig(c)) } -func runManagers(t *testing.T, ctx context.Context, opts *Options, app storage.Appendable) (*discovery.Manager, *Manager) { +func runManagers(t *testing.T, ctx context.Context, opts *Options) (*discovery.Manager, *Manager, *teststorage.Appender) { t.Helper() if opts == nil { opts = &Options{} } opts.DiscoveryReloadInterval = model.Duration(100 * time.Millisecond) - if app == nil { - app = nopAppendable{} - } + + appTest := teststorage.NewAppender() reg := prometheus.NewRegistry() sdMetrics, err := discovery.RegisterSDMetrics(reg, discovery.NewRefreshMetrics(reg)) @@ -1216,17 +971,17 @@ func runManagers(t *testing.T, ctx context.Context, opts *Options, app storage.A sdMetrics, discovery.Updatert(100*time.Millisecond), ) - scrapeManager, err := NewManager( + scrapeManager, err := NewManagerWithAppendableV2( opts, nil, nil, - app, + appTest, prometheus.NewRegistry(), ) require.NoError(t, err) go discoveryManager.Run() go scrapeManager.Run(discoveryManager.SyncCh()) - return discoveryManager, scrapeManager + return discoveryManager, scrapeManager, appTest } func writeIntoFile(t *testing.T, content, filePattern string) *os.File { @@ -1293,7 +1048,7 @@ scrape_configs: - files: ['%s'] ` - discoveryManager, scrapeManager := runManagers(t, ctx, nil, nil) + discoveryManager, scrapeManager, _ := runManagers(t, ctx, nil) defer scrapeManager.Stop() applyConfig( @@ -1392,7 +1147,7 @@ scrape_configs: file_sd_configs: - files: ['%s', '%s'] ` - discoveryManager, scrapeManager := runManagers(t, ctx, nil, nil) + discoveryManager, scrapeManager, _ := runManagers(t, ctx, nil) defer scrapeManager.Stop() applyConfig( @@ -1451,7 +1206,7 @@ scrape_configs: file_sd_configs: - files: ['%s'] ` - discoveryManager, scrapeManager := runManagers(t, ctx, nil, nil) + discoveryManager, scrapeManager, _ := runManagers(t, ctx, nil) defer scrapeManager.Stop() applyConfig( @@ -1517,7 +1272,7 @@ scrape_configs: - targets: ['%s'] ` - discoveryManager, scrapeManager := runManagers(t, ctx, nil, nil) + discoveryManager, scrapeManager, _ := runManagers(t, ctx, nil) defer scrapeManager.Stop() // Apply the initial config with an existing file @@ -1601,7 +1356,7 @@ scrape_configs: cfg := loadConfiguration(t, cfgText) - m, err := NewManager(&Options{}, nil, nil, &nopAppendable{}, prometheus.NewRegistry()) + m, err := NewManagerWithAppendableV2(&Options{}, nil, nil, &nopAppendable{}, prometheus.NewRegistry()) require.NoError(t, err) defer m.Stop() require.NoError(t, m.ApplyConfig(cfg)) diff --git a/scrape/scrape.go b/scrape/scrape.go index bbb93c8801..2cd62cf069 100644 --- a/scrape/scrape.go +++ b/scrape/scrape.go @@ -80,10 +80,12 @@ type FailureLogger interface { // scrapePool manages scrapes for sets of targets. type scrapePool struct { - appendable storage.Appendable - logger *slog.Logger - cancel context.CancelFunc - httpOpts []config_util.HTTPClientOption + appendableV1 storage.Appendable + appendableV2 storage.AppendableV2 + + logger *slog.Logger + cancel context.CancelFunc + httpOpts []config_util.HTTPClientOption // mtx must not be taken after targetMtx. mtx sync.Mutex @@ -147,7 +149,14 @@ const maxAheadTime = 10 * time.Minute // returning an empty label set is interpreted as "drop". type labelsMutator func(labels.Labels) labels.Labels -func newScrapePool(cfg *config.ScrapeConfig, app storage.Appendable, offsetSeed uint64, logger *slog.Logger, buffers *pool.Pool, options *Options, metrics *scrapeMetrics) (*scrapePool, error) { +type scrapeLoopAppender interface { + storage.AppenderTransaction + + addReportSample(s reportSample, t int64, v float64, b *labels.Builder, rejectOOO bool) error + append(b []byte, contentType string, ts time.Time) (total, added, seriesAdded int, err error) +} + +func newScrapePool(cfg *config.ScrapeConfig, appendableV1 storage.Appendable, appendableV2 storage.AppendableV2, offsetSeed uint64, logger *slog.Logger, buffers *pool.Pool, options *Options, metrics *scrapeMetrics) (*scrapePool, error) { if logger == nil { logger = promslog.NewNopLogger() } @@ -169,7 +178,8 @@ func newScrapePool(cfg *config.ScrapeConfig, app storage.Appendable, offsetSeed ctx, cancel := context.WithCancel(context.Background()) sp := &scrapePool{ cancel: cancel, - appendable: app, + appendableV1: appendableV1, + appendableV2: appendableV2, config: cfg, client: client, activeTargets: map[uint64]*Target{}, @@ -183,55 +193,76 @@ func newScrapePool(cfg *config.ScrapeConfig, app storage.Appendable, offsetSeed escapingScheme: escapingScheme, } sp.newLoop = func(opts scrapeLoopOptions) loop { - // Update the targets retrieval function for metadata to a new scrape cache. - cache := opts.cache - if cache == nil { - cache = newScrapeCache(metrics) - } - opts.target.SetMetadataStore(cache) - - return newScrapeLoop( - ctx, - opts.scraper, - logger.With("target", opts.target), - buffers, - func(l labels.Labels) labels.Labels { + // NOTE: Formatting matches scrapeLoop fields order for readability. + sl := &scrapeLoop{ + buffers: buffers, + appendableV1: appendableV1, + appendableV2: appendableV2, + sampleMutator: func(l labels.Labels) labels.Labels { return mutateSampleLabels(l, opts.target, opts.honorLabels, opts.mrc) }, - func(l labels.Labels) labels.Labels { return mutateReportSampleLabels(l, opts.target) }, - func(ctx context.Context) storage.Appender { return app.Appender(ctx) }, - cache, - sp.symbolTable, - offsetSeed, - opts.honorTimestamps, - opts.trackTimestampsStaleness, - opts.enableCompression, - opts.sampleLimit, - opts.bucketLimit, - opts.maxSchema, - opts.labelLimits, - opts.interval, - opts.timeout, - opts.alwaysScrapeClassicHist, - opts.convertClassicHistToNHCB, - cfg.ScrapeNativeHistogramsEnabled(), - options.EnableStartTimestampZeroIngestion, - options.EnableTypeAndUnitLabels, - options.ExtraMetrics, - options.AppendMetadata, - opts.target, - options.PassMetadataInContext, - metrics, - options.skipOffsetting, - sp.validationScheme, - sp.escapingScheme, - opts.fallbackScrapeProtocol, - ) + reportSampleMutator: func(l labels.Labels) labels.Labels { return mutateReportSampleLabels(l, opts.target) }, + offsetSeed: offsetSeed, + metrics: metrics, + + symbolTable: sp.symbolTable, + validationScheme: sp.validationScheme, + escapingScheme: sp.escapingScheme, + + enableNativeHistogramScraping: cfg.ScrapeNativeHistogramsEnabled(), + + enableSTZeroIngestion: options.EnableStartTimestampZeroIngestion, + enableTypeAndUnitLabels: options.EnableTypeAndUnitLabels, + reportExtraMetrics: options.ExtraMetrics, + appendMetadataToWAL: options.AppendMetadata, + skipOffsetting: options.skipOffsetting, + + scrapeLoopOptions: opts, + } + sl.init(ctx, options.PassMetadataInContext) + return sl } sp.metrics.targetScrapePoolTargetLimit.WithLabelValues(sp.config.JobName).Set(float64(sp.config.TargetLimit)) return sp, nil } +// init prepares scrapeLoop after raw construction. +// NOTE: While newScrapeLoop constructor pattern would be safer, it has proven to be +// highly not readable (too many params). Instead, we follow init pattern. +func (sl *scrapeLoop) init(ctx context.Context, passMetadataInContext bool) { + if sl.l == nil { + sl.l = promslog.NewNopLogger() + } + sl.parentCtx = ctx + sl.stopped = make(chan struct{}) + if sl.buffers == nil { + sl.buffers = pool.New(1e3, 1e6, 3, func(sz int) any { return make([]byte, 0, sz) }) + } + if sl.cache == nil { + sl.cache = newScrapeCache(sl.metrics) + if sl.target != nil { + // Update the targets retrieval function for metadata to a new scrape cache. + sl.target.SetMetadataStore(sl.cache) + // TODO(bwplotka): Not sure why, but doing this before sl.target.SetMetadataStore(sl.cache) blocks goroutines... + // Debug, something is odd. + sl.l = sl.l.With("target", sl.target) + } + } + + appenderCtx := ctx + if passMetadataInContext { + // Store the cache and target in the context. This is then used by downstream OTel Collector + // to lookup the metadata required to process the samples. Not used by Prometheus itself. + // TODO(gouthamve) We're using a dedicated context because using the parentCtx caused a memory + // leak. We should ideally fix the main leak. See: https://github.com/prometheus/prometheus/pull/10590 + // TODO(bwplotka): Remove once OpenTelemetry collector uses AppenderV2 (add issue) + appenderCtx = ContextWithMetricMetadataStore(appenderCtx, sl.cache) + appenderCtx = ContextWithTarget(appenderCtx, sl.target) + } + sl.appenderCtx = appenderCtx + sl.ctx, sl.cancel = context.WithCancel(ctx) +} + func (sp *scrapePool) ActiveTargets() []*Target { sp.targetMtx.Lock() defer sp.targetMtx.Unlock() @@ -392,6 +423,8 @@ func (sp *scrapePool) restartLoops(reuseCache bool) { } t := sp.activeTargets[fp] + // Update the targets retrieval function for metadata to a new target. + t.SetMetadataStore(cache) targetInterval, targetTimeout, err := t.intervalAndTimeout(interval, timeout) var ( s = &targetScraper{ @@ -753,39 +786,6 @@ func mutateReportSampleLabels(lset labels.Labels, target *Target) labels.Labels return lb.Labels() } -// appender returns an appender for ingested samples from the target. -func appender(app storage.Appender, sampleLimit, bucketLimit int, maxSchema int32) storage.Appender { - app = &timeLimitAppender{ - Appender: app, - maxTime: timestamp.FromTime(time.Now().Add(maxAheadTime)), - } - - // The sampleLimit is applied after metrics are potentially dropped via relabeling. - if sampleLimit > 0 { - app = &limitAppender{ - Appender: app, - limit: sampleLimit, - } - } - - if bucketLimit > 0 { - app = &bucketLimitAppender{ - Appender: app, - limit: bucketLimit, - } - } - - if maxSchema < histogram.ExponentialSchemaMax { - app = &maxSchemaAppender{ - Appender: app, - maxSchema: maxSchema, - } - } - - return app -} - -// A scraper retrieves samples and accepts a status report at the end. type scraper interface { scrape(ctx context.Context) (*http.Response, error) readResponse(ctx context.Context, resp *http.Response, w io.Writer) (string, error) @@ -931,55 +931,50 @@ type cacheEntry struct { } type scrapeLoop struct { - scraper scraper - l *slog.Logger - scrapeFailureLogger FailureLogger - scrapeFailureLoggerMtx sync.RWMutex - cache *scrapeCache - lastScrapeSize int - buffers *pool.Pool - offsetSeed uint64 - honorTimestamps bool - trackTimestampsStaleness bool - enableCompression bool - forcedErr error - forcedErrMtx sync.Mutex - sampleLimit int - bucketLimit int - maxSchema int32 - labelLimits *labelLimits - interval time.Duration - timeout time.Duration - validationScheme model.ValidationScheme - escapingScheme model.EscapingScheme - - alwaysScrapeClassicHist bool - convertClassicHistToNHCB bool - enableSTZeroIngestion bool - enableTypeAndUnitLabels bool - fallbackScrapeProtocol string - - enableNativeHistogramScraping bool - - appender func(ctx context.Context) storage.Appender - symbolTable *labels.SymbolTable + // Parameters. + ctx context.Context + cancel func() + stopped chan struct{} + parentCtx context.Context + appenderCtx context.Context + l *slog.Logger + buffers *pool.Pool + appendableV1 storage.Appendable + appendableV2 storage.AppendableV2 sampleMutator labelsMutator reportSampleMutator labelsMutator + offsetSeed uint64 + metrics *scrapeMetrics - parentCtx context.Context - appenderCtx context.Context - ctx context.Context - cancel func() - stopped chan struct{} + // Scrape pool shared data. + symbolTable *labels.SymbolTable + validationScheme model.ValidationScheme + escapingScheme model.EscapingScheme + // Options inherited from config.ScrapeConfig. + enableNativeHistogramScraping bool + + // Options inherited from scrape.Options. + enableSTZeroIngestion bool + enableTypeAndUnitLabels bool + reportExtraMetrics bool + appendMetadataToWAL bool + skipOffsetting bool // For testability. + + // Common options. + scrapeLoopOptions + + // error injection through setForcedError. + forcedErr error + forcedErrMtx sync.Mutex + + // Special logger set on setScrapeFailureLogger + scrapeFailureLoggerMtx sync.RWMutex + scrapeFailureLogger FailureLogger + + // Locally cached data. + lastScrapeSize int disabledEndOfRunStalenessMarkers atomic.Bool - - reportExtraMetrics bool - appendMetadataToWAL bool - - metrics *scrapeMetrics - - skipOffsetting bool // For testability. } // scrapeCache tracks mappings of exposed metric strings to label sets and @@ -1004,8 +999,8 @@ type scrapeCache struct { seriesCur map[storage.SeriesRef]*cacheEntry seriesPrev map[storage.SeriesRef]*cacheEntry - // TODO(bwplotka): Consider moving Metadata API to use WAL instead of scrape loop to - // avoid locking (using metadata API can block scraping). + // TODO(bwplotka): Consider moving metadata caching to head. See + // https://github.com/prometheus/prometheus/issues/17619. metaMtx sync.Mutex // Mutex is needed due to api touching it when metadata is queried. metadata map[string]*metaEntry // metadata by metric family name. @@ -1240,101 +1235,6 @@ func (c *scrapeCache) LengthMetadata() int { return len(c.metadata) } -func newScrapeLoop(ctx context.Context, - sc scraper, - l *slog.Logger, - buffers *pool.Pool, - sampleMutator labelsMutator, - reportSampleMutator labelsMutator, - appender func(ctx context.Context) storage.Appender, - cache *scrapeCache, - symbolTable *labels.SymbolTable, - offsetSeed uint64, - honorTimestamps bool, - trackTimestampsStaleness bool, - enableCompression bool, - sampleLimit int, - bucketLimit int, - maxSchema int32, - labelLimits *labelLimits, - interval time.Duration, - timeout time.Duration, - alwaysScrapeClassicHist bool, - convertClassicHistToNHCB bool, - enableNativeHistogramScraping bool, - enableSTZeroIngestion bool, - enableTypeAndUnitLabels bool, - reportExtraMetrics bool, - appendMetadataToWAL bool, - target *Target, - passMetadataInContext bool, - metrics *scrapeMetrics, - skipOffsetting bool, - validationScheme model.ValidationScheme, - escapingScheme model.EscapingScheme, - fallbackScrapeProtocol string, -) *scrapeLoop { - if l == nil { - l = promslog.NewNopLogger() - } - if buffers == nil { - buffers = pool.New(1e3, 1e6, 3, func(sz int) any { return make([]byte, 0, sz) }) - } - if cache == nil { - cache = newScrapeCache(metrics) - } - - appenderCtx := ctx - - if passMetadataInContext { - // Store the cache and target in the context. This is then used by downstream OTel Collector - // to lookup the metadata required to process the samples. Not used by Prometheus itself. - // TODO(gouthamve) We're using a dedicated context because using the parentCtx caused a memory - // leak. We should ideally fix the main leak. See: https://github.com/prometheus/prometheus/pull/10590 - appenderCtx = ContextWithMetricMetadataStore(appenderCtx, cache) - appenderCtx = ContextWithTarget(appenderCtx, target) - } - - sl := &scrapeLoop{ - scraper: sc, - buffers: buffers, - cache: cache, - appender: appender, - symbolTable: symbolTable, - sampleMutator: sampleMutator, - reportSampleMutator: reportSampleMutator, - stopped: make(chan struct{}), - offsetSeed: offsetSeed, - l: l, - parentCtx: ctx, - appenderCtx: appenderCtx, - honorTimestamps: honorTimestamps, - trackTimestampsStaleness: trackTimestampsStaleness, - enableCompression: enableCompression, - sampleLimit: sampleLimit, - bucketLimit: bucketLimit, - maxSchema: maxSchema, - labelLimits: labelLimits, - interval: interval, - timeout: timeout, - alwaysScrapeClassicHist: alwaysScrapeClassicHist, - convertClassicHistToNHCB: convertClassicHistToNHCB, - enableSTZeroIngestion: enableSTZeroIngestion, - enableTypeAndUnitLabels: enableTypeAndUnitLabels, - fallbackScrapeProtocol: fallbackScrapeProtocol, - enableNativeHistogramScraping: enableNativeHistogramScraping, - reportExtraMetrics: reportExtraMetrics, - appendMetadataToWAL: appendMetadataToWAL, - metrics: metrics, - skipOffsetting: skipOffsetting, - validationScheme: validationScheme, - escapingScheme: escapingScheme, - } - sl.ctx, sl.cancel = context.WithCancel(ctx) - - return sl -} - func (sl *scrapeLoop) setScrapeFailureLogger(l FailureLogger) { sl.scrapeFailureLoggerMtx.Lock() defer sl.scrapeFailureLoggerMtx.Unlock() @@ -1411,6 +1311,13 @@ mainLoop: } } +func (sl *scrapeLoop) appender() scrapeLoopAppender { + if sl.appendableV2 != nil { + return &scrapeLoopAppenderV2{scrapeLoop: sl, AppenderV2: sl.appendableV2.AppenderV2(sl.appenderCtx)} + } + return &scrapeLoopAppenderV1{scrapeLoop: sl, Appender: sl.appendableV1.Appender(sl.appenderCtx)} +} + // scrapeAndReport performs a scrape and then appends the result to the storage // together with reporting metrics, by using as few appenders as possible. // In the happy scenario, a single appender is used. @@ -1432,20 +1339,20 @@ func (sl *scrapeLoop) scrapeAndReport(last, appendTime time.Time, errc chan<- er var total, added, seriesAdded, bytesRead int var err, appErr, scrapeErr error - app := sl.appender(sl.appenderCtx) + sla := sl.appender() defer func() { if err != nil { - app.Rollback() + _ = sla.Rollback() return } - err = app.Commit() + err = sla.Commit() if err != nil { sl.l.Error("Scrape commit failed", "err", err) } }() defer func() { - if err = sl.report(app, appendTime, time.Since(start), total, added, seriesAdded, bytesRead, scrapeErr); err != nil { + if err = sl.report(sla, appendTime, time.Since(start), total, added, seriesAdded, bytesRead, scrapeErr); err != nil { sl.l.Warn("Appending scrape report failed", "err", err) } }() @@ -1453,9 +1360,9 @@ func (sl *scrapeLoop) scrapeAndReport(last, appendTime time.Time, errc chan<- er if forcedErr := sl.getForcedError(); forcedErr != nil { scrapeErr = forcedErr // Add stale markers. - if _, _, _, err := sl.append(app, []byte{}, "", appendTime); err != nil { - app.Rollback() - app = sl.appender(sl.appenderCtx) + if _, _, _, err := sla.append([]byte{}, "", appendTime); err != nil { + _ = sla.Rollback() + sla = sl.appender() sl.l.Warn("Append failed", "err", err) } if errc != nil { @@ -1505,16 +1412,16 @@ func (sl *scrapeLoop) scrapeAndReport(last, appendTime time.Time, errc chan<- er // A failed scrape is the same as an empty scrape, // we still call sl.append to trigger stale markers. - total, added, seriesAdded, appErr = sl.append(app, b, contentType, appendTime) + total, added, seriesAdded, appErr = sla.append(b, contentType, appendTime) if appErr != nil { - app.Rollback() - app = sl.appender(sl.appenderCtx) + _ = sla.Rollback() + sla = sl.appender() sl.l.Debug("Append failed", "err", appErr) // The append failed, probably due to a parse error or sample limit. // Call sl.append again with an empty scrape to trigger stale markers. - if _, _, _, err := sl.append(app, []byte{}, "", appendTime); err != nil { - app.Rollback() - app = sl.appender(sl.appenderCtx) + if _, _, _, err := sla.append([]byte{}, "", appendTime); err != nil { + _ = sla.Rollback() + sla = sl.appender() sl.l.Warn("Append failed", "err", err) } } @@ -1584,24 +1491,24 @@ func (sl *scrapeLoop) endOfRunStaleness(last time.Time, ticker *time.Ticker, int // If the target has since been recreated and scraped, the // stale markers will be out of order and ignored. // sl.context would have been cancelled, hence using sl.appenderCtx. - app := sl.appender(sl.appenderCtx) + sla := sl.appender() var err error defer func() { if err != nil { - app.Rollback() + _ = sla.Rollback() return } - err = app.Commit() + err = sla.Commit() if err != nil { sl.l.Warn("Stale commit failed", "err", err) } }() - if _, _, _, err = sl.append(app, []byte{}, "", staleTime); err != nil { - app.Rollback() - app = sl.appender(sl.appenderCtx) + if _, _, _, err = sla.append([]byte{}, "", staleTime); err != nil { + _ = sla.Rollback() + sla = sl.appender() sl.l.Warn("Stale append failed", "err", err) } - if err = sl.reportStale(app, staleTime); err != nil { + if err = sl.reportStale(sla, staleTime); err != nil { sl.l.Warn("Stale report failed", "err", err) } } @@ -1629,12 +1536,11 @@ type appendErrors struct { } // Update the stale markers. -func (sl *scrapeLoop) updateStaleMarkers(app storage.Appender, defTime int64) (err error) { +func (sl *scrapeLoop) updateStaleMarkers(app storage.AppenderV2, defTime int64) (err error) { sl.cache.forEachStale(func(ref storage.SeriesRef, lset labels.Labels) bool { // Series no longer exposed, mark it stale. - app.SetOptions(&storage.AppendOptions{DiscardOutOfOrder: true}) - _, err = app.Append(ref, lset, defTime, math.Float64frombits(value.StaleNaN)) - app.SetOptions(nil) + // TODO(bwplotka): Pass through Metadata and MFName? + _, err = app.Append(ref, lset, 0, defTime, math.Float64frombits(value.StaleNaN), nil, nil, storage.AOptions{RejectOutOfOrder: true}) switch { case errors.Is(err, storage.ErrOutOfOrderSample), errors.Is(err, storage.ErrDuplicateSampleForTimestamp): // Do not count these in logging, as this is expected if a target @@ -1646,12 +1552,20 @@ func (sl *scrapeLoop) updateStaleMarkers(app storage.Appender, defTime int64) (e return err } -func (sl *scrapeLoop) append(app storage.Appender, b []byte, contentType string, ts time.Time) (total, added, seriesAdded int, err error) { +type scrapeLoopAppenderV2 struct { + *scrapeLoop + + storage.AppenderV2 +} + +var _ scrapeLoopAppender = &scrapeLoopAppenderV2{} + +func (sl *scrapeLoopAppenderV2) append(b []byte, contentType string, ts time.Time) (total, added, seriesAdded int, err error) { defTime := timestamp.FromTime(ts) if len(b) == 0 { // Empty scrape. Just update the stale makers and swap the cache (but don't flush it). - err = sl.updateStaleMarkers(app, defTime) + err = sl.updateStaleMarkers(sl.AppenderV2, defTime) sl.cache.iterDone(false) return total, added, seriesAdded, err } @@ -1689,13 +1603,11 @@ func (sl *scrapeLoop) append(app storage.Appender, b []byte, contentType string, e exemplar.Exemplar // Escapes to heap so hoisted out of loop. lastMeta *metaEntry lastMFName []byte + exemplars = make([]exemplar.Exemplar, 0, 1) ) - exemplars := make([]exemplar.Exemplar, 0, 1) - // Take an appender with limits. - app = appender(app, sl.sampleLimit, sl.bucketLimit, sl.maxSchema) - + app := appender(sl.AppenderV2, sl.sampleLimit, sl.bucketLimit, sl.maxSchema) defer func() { if err != nil { return @@ -1783,7 +1695,7 @@ loop: continue } - if !lset.Has(labels.MetricName) { + if !lset.Has(model.MetricNameLabel) { err = errNameLabelMandatory break loop } @@ -1802,53 +1714,79 @@ loop: if seriesAlreadyScraped && parsedTimestamp == nil { err = storage.ErrDuplicateSampleForTimestamp } else { + st := int64(0) if sl.enableSTZeroIngestion { - if stMs := p.StartTimestamp(); stMs != 0 { + // p.StartTimestamp tend to be expensive (e.g. OM1) do it only if we care. + st = p.StartTimestamp() + } + + exemplars = exemplars[:0] // Reset and reuse the exemplar slice. + for hasExemplar := p.Exemplar(&e); hasExemplar; hasExemplar = p.Exemplar(&e) { + if !e.HasTs { if isHistogram { - if h != nil { - ref, err = app.AppendHistogramSTZeroSample(ref, lset, t, stMs, h, nil) - } else { - ref, err = app.AppendHistogramSTZeroSample(ref, lset, t, stMs, nil, fh) - } - } else { - ref, err = app.AppendSTZeroSample(ref, lset, t, stMs) + // We drop exemplars for native histograms if they don't have a timestamp. + // Missing timestamps are deliberately not supported as we want to start + // enforcing timestamps for exemplars as otherwise proper deduplication + // is inefficient and purely based on heuristics: we cannot distinguish + // between repeated exemplars and new instances with the same values. + // This is done silently without logs as it is not an error but out of spec. + // This does not affect classic histograms so that behaviour is unchanged. + e = exemplar.Exemplar{} // Reset for next time round loop. + continue } - if err != nil && !errors.Is(err, storage.ErrOutOfOrderST) { // OOO is a common case, ignoring completely for now. - // ST is an experimental feature. For now, we don't need to fail the - // scrape on errors updating the created timestamp, log debug. - sl.l.Debug("Error when appending ST in scrape loop", "series", string(met), "ct", stMs, "t", t, "err", err) + e.Ts = t + } + exemplars = append(exemplars, e) + e = exemplar.Exemplar{} // Reset for next time round loop. + } + + // Prepare append call. + appOpts := storage.AOptions{ + MetricFamilyName: yoloString(lastMFName), + } + if len(exemplars) > 0 { + // Sort so that checking for duplicates / out of order is more efficient during validation. + // TODO(bwplotka): Double check if this is even true now. + slices.SortFunc(exemplars, exemplar.Compare) + appOpts.Exemplars = exemplars + } + + // TODO(bwplotka): This mimicking the scrape appender v1 flow. Once we remove v1 + // flow we should rename appendMetadataToWAL flag to passMetadata because at this + // point if the metadata is appended to WAL or only to memory or anything else is + // completely up to the implementation. All known implementations (Prom and Otel) also + // support always passing metadata (e.g. Prometheus head memSeries.metadata + // can help with detection, no need to detect and pass it only if changed. + if sl.appendMetadataToWAL && lastMeta != nil { + if !seriesCached || lastMeta.lastIterChange != sl.cache.iter { + // In majority cases we can trust that the current series/histogram is matching the lastMeta and lastMFName. + // However, optional TYPE etc metadata and broken OM text can break this, detect those cases here. + // TODO(bwplotka): Consider moving this to parser as many parser users end up doing this (e.g. ST and NHCB parsing). + if !isSeriesPartOfFamily(lset.Get(model.MetricNameLabel), lastMFName, lastMeta.Type) { + lastMeta = nil } } + if lastMeta != nil { + appOpts.Metadata = lastMeta.Metadata + } } - if isHistogram { - if h != nil { - ref, err = app.AppendHistogram(ref, lset, t, h, nil) - } else { - ref, err = app.AppendHistogram(ref, lset, t, nil, fh) - } - } else { - ref, err = app.Append(ref, lset, t, val) - } + // Append sample to the storage. + ref, err = app.Append(ref, lset, st, t, val, h, fh, appOpts) } - - if err == nil { - if (parsedTimestamp == nil || sl.trackTimestampsStaleness) && ce != nil { - sl.cache.trackStaleness(ce.ref, ce) - } - } - - sampleAdded, err = sl.checkAddError(met, err, &sampleLimitErr, &bucketLimitErr, &appErrs) + sampleAdded, err = sl.checkAddError(met, exemplars, err, &sampleLimitErr, &bucketLimitErr, &appErrs) if err != nil { if !errors.Is(err, storage.ErrNotFound) { sl.l.Debug("Unexpected error", "series", string(met), "err", err) } break loop + } else if (parsedTimestamp == nil || sl.trackTimestampsStaleness) && ce != nil { + sl.cache.trackStaleness(ce.ref, ce) } - // If series wasn't cached (is new, not seen on previous scrape) we need need to add it to the scrape cache. + // If series wasn't cached (is new, not seen on previous scrape) we need to add it to the scrape cache. // But we only do this for series that were appended to TSDB without errors. - // If a series was new but we didn't append it due to sample_limit or other errors then we don't need + // If a series was new, but we didn't append it due to sample_limit or other errors then we don't need // it in the scrape cache because we don't need to emit StaleNaNs for it when it disappears. if !seriesCached && sampleAdded { ce = sl.cache.addRef(met, ref, lset, hash) @@ -1857,7 +1795,7 @@ loop: // But make sure we only do this if we have a cache entry (ce) for our series. sl.cache.trackStaleness(ref, ce) } - if sampleAdded && sampleLimitErr == nil && bucketLimitErr == nil { + if sampleLimitErr == nil && bucketLimitErr == nil { seriesAdded++ } } @@ -1867,62 +1805,6 @@ loop: // We still report duplicated samples here since this number should be the exact number // of time series exposed on a scrape after relabelling. added++ - exemplars = exemplars[:0] // Reset and reuse the exemplar slice. - for hasExemplar := p.Exemplar(&e); hasExemplar; hasExemplar = p.Exemplar(&e) { - if !e.HasTs { - if isHistogram { - // We drop exemplars for native histograms if they don't have a timestamp. - // Missing timestamps are deliberately not supported as we want to start - // enforcing timestamps for exemplars as otherwise proper deduplication - // is inefficient and purely based on heuristics: we cannot distinguish - // between repeated exemplars and new instances with the same values. - // This is done silently without logs as it is not an error but out of spec. - // This does not affect classic histograms so that behaviour is unchanged. - e = exemplar.Exemplar{} // Reset for next time round loop. - continue - } - e.Ts = t - } - exemplars = append(exemplars, e) - e = exemplar.Exemplar{} // Reset for next time round loop. - } - // Sort so that checking for duplicates / out of order is more efficient during validation. - slices.SortFunc(exemplars, exemplar.Compare) - outOfOrderExemplars := 0 - for _, e := range exemplars { - _, exemplarErr := app.AppendExemplar(ref, lset, e) - switch { - case exemplarErr == nil: - // Do nothing. - case errors.Is(exemplarErr, storage.ErrOutOfOrderExemplar): - outOfOrderExemplars++ - default: - // Since exemplar storage is still experimental, we don't fail the scrape on ingestion errors. - sl.l.Debug("Error while adding exemplar in AddExemplar", "exemplar", fmt.Sprintf("%+v", e), "err", exemplarErr) - } - } - if outOfOrderExemplars > 0 && outOfOrderExemplars == len(exemplars) { - // Only report out of order exemplars if all are out of order, otherwise this was a partial update - // to some existing set of exemplars. - appErrs.numExemplarOutOfOrder += outOfOrderExemplars - sl.l.Debug("Out of order exemplars", "count", outOfOrderExemplars, "latest", fmt.Sprintf("%+v", exemplars[len(exemplars)-1])) - sl.metrics.targetScrapeExemplarOutOfOrder.Add(float64(outOfOrderExemplars)) - } - - if sl.appendMetadataToWAL && lastMeta != nil { - // Is it new series OR did metadata change for this family? - if !seriesCached || lastMeta.lastIterChange == sl.cache.iter { - // In majority cases we can trust that the current series/histogram is matching the lastMeta and lastMFName. - // However, optional TYPE etc metadata and broken OM text can break this, detect those cases here. - // TODO(bwplotka): Consider moving this to parser as many parser users end up doing this (e.g. ST and NHCB parsing). - if isSeriesPartOfFamily(lset.Get(labels.MetricName), lastMFName, lastMeta.Type) { - if _, merr := app.UpdateMetadata(ref, lset, lastMeta.Metadata); merr != nil { - // No need to fail the scrape on errors appending metadata. - sl.l.Debug("Error when appending metadata in scrape loop", "ref", fmt.Sprintf("%d", ref), "metadata", fmt.Sprintf("%+v", lastMeta.Metadata), "err", merr) - } - } - } - } } if sampleLimitErr != nil { if err == nil { @@ -1956,6 +1838,38 @@ loop: return total, added, seriesAdded, err } +// appender returns an appender for ingested samples from the target. +func appender(app storage.AppenderV2, sampleLimit, bucketLimit int, maxSchema int32) storage.AppenderV2 { + app = &timeLimitAppender{ + AppenderV2: app, + maxTime: timestamp.FromTime(time.Now().Add(maxAheadTime)), + } + + // The sampleLimit is applied after metrics are potentially dropped via relabeling. + if sampleLimit > 0 { + app = &limitAppender{ + AppenderV2: app, + limit: sampleLimit, + } + } + + if bucketLimit > 0 { + app = &bucketLimitAppender{ + AppenderV2: app, + limit: bucketLimit, + } + } + + if maxSchema < histogram.ExponentialSchemaMax { + app = &maxSchemaAppender{ + AppenderV2: app, + maxSchema: maxSchema, + } + } + + return app +} + func isSeriesPartOfFamily(mName string, mfName []byte, typ model.MetricType) bool { mfNameStr := yoloString(mfName) if !strings.HasPrefix(mName, mfNameStr) { // Fast path. @@ -2027,7 +1941,8 @@ func isSeriesPartOfFamily(mName string, mfName []byte, typ model.MetricType) boo // during normal operation (e.g., accidental cardinality explosion, sudden traffic spikes). // Current case ordering prevents exercising other cases when limits are exceeded. // Remaining error cases typically occur only a few times, often during initial setup. -func (sl *scrapeLoop) checkAddError(met []byte, err error, sampleLimitErr, bucketLimitErr *error, appErrs *appendErrors) (bool, error) { +func (sl *scrapeLoop) checkAddError(met []byte, exemplars []exemplar.Exemplar, err error, sampleLimitErr, bucketLimitErr *error, appErrs *appendErrors) (sampleAdded bool, _ error) { + var pErr *storage.AppendPartialError switch { case err == nil: return true, nil @@ -2058,6 +1973,23 @@ func (sl *scrapeLoop) checkAddError(met []byte, err error, sampleLimitErr, bucke return false, nil case errors.Is(err, storage.ErrNotFound): return false, storage.ErrNotFound + case errors.As(err, &pErr): + outOfOrderExemplars := 0 + for _, e := range pErr.ExemplarErrors { + if errors.Is(e, storage.ErrOutOfOrderExemplar) { + outOfOrderExemplars++ + } + // Since exemplar storage is still experimental, we don't fail or check other errors. + // Debug log is emmited in TSDB already. + } + if outOfOrderExemplars > 0 && outOfOrderExemplars == len(exemplars) { + // Only report out of order exemplars if all are out of order, otherwise this was a partial update + // to some existing set of exemplars. + appErrs.numExemplarOutOfOrder += outOfOrderExemplars + sl.l.Debug("Out of order exemplars", "count", outOfOrderExemplars, "latest", fmt.Sprintf("%+v", exemplars[len(exemplars)-1])) + sl.metrics.targetScrapeExemplarOutOfOrder.Add(float64(outOfOrderExemplars)) + } + return true, nil default: return false, err } @@ -2139,7 +2071,7 @@ var ( } ) -func (sl *scrapeLoop) report(app storage.Appender, start time.Time, duration time.Duration, scraped, added, seriesAdded, bytes int, scrapeErr error) (err error) { +func (sl *scrapeLoop) report(sla scrapeLoopAppender, start time.Time, duration time.Duration, scraped, added, seriesAdded, bytes int, scrapeErr error) (err error) { sl.scraper.Report(start, duration, scrapeErr) ts := timestamp.FromTime(start) @@ -2150,71 +2082,70 @@ func (sl *scrapeLoop) report(app storage.Appender, start time.Time, duration tim } b := labels.NewBuilderWithSymbolTable(sl.symbolTable) - if err = sl.addReportSample(app, scrapeHealthMetric, ts, health, b); err != nil { + if err = sla.addReportSample(scrapeHealthMetric, ts, health, b, false); err != nil { return err } - if err = sl.addReportSample(app, scrapeDurationMetric, ts, duration.Seconds(), b); err != nil { + if err = sla.addReportSample(scrapeDurationMetric, ts, duration.Seconds(), b, false); err != nil { return err } - if err = sl.addReportSample(app, scrapeSamplesMetric, ts, float64(scraped), b); err != nil { + if err = sla.addReportSample(scrapeSamplesMetric, ts, float64(scraped), b, false); err != nil { return err } - if err = sl.addReportSample(app, samplesPostRelabelMetric, ts, float64(added), b); err != nil { + if err = sla.addReportSample(samplesPostRelabelMetric, ts, float64(added), b, false); err != nil { return err } - if err = sl.addReportSample(app, scrapeSeriesAddedMetric, ts, float64(seriesAdded), b); err != nil { + if err = sla.addReportSample(scrapeSeriesAddedMetric, ts, float64(seriesAdded), b, false); err != nil { return err } if sl.reportExtraMetrics { - if err = sl.addReportSample(app, scrapeTimeoutMetric, ts, sl.timeout.Seconds(), b); err != nil { + if err = sla.addReportSample(scrapeTimeoutMetric, ts, sl.timeout.Seconds(), b, false); err != nil { return err } - if err = sl.addReportSample(app, scrapeSampleLimitMetric, ts, float64(sl.sampleLimit), b); err != nil { + if err = sla.addReportSample(scrapeSampleLimitMetric, ts, float64(sl.sampleLimit), b, false); err != nil { return err } - if err = sl.addReportSample(app, scrapeBodySizeBytesMetric, ts, float64(bytes), b); err != nil { + if err = sla.addReportSample(scrapeBodySizeBytesMetric, ts, float64(bytes), b, false); err != nil { return err } } return err } -func (sl *scrapeLoop) reportStale(app storage.Appender, start time.Time) (err error) { +func (sl *scrapeLoop) reportStale(sla scrapeLoopAppender, start time.Time) (err error) { ts := timestamp.FromTime(start) - app.SetOptions(&storage.AppendOptions{DiscardOutOfOrder: true}) stale := math.Float64frombits(value.StaleNaN) b := labels.NewBuilder(labels.EmptyLabels()) - if err = sl.addReportSample(app, scrapeHealthMetric, ts, stale, b); err != nil { + if err = sla.addReportSample(scrapeHealthMetric, ts, stale, b, true); err != nil { return err } - if err = sl.addReportSample(app, scrapeDurationMetric, ts, stale, b); err != nil { + if err = sla.addReportSample(scrapeDurationMetric, ts, stale, b, true); err != nil { return err } - if err = sl.addReportSample(app, scrapeSamplesMetric, ts, stale, b); err != nil { + if err = sla.addReportSample(scrapeSamplesMetric, ts, stale, b, true); err != nil { return err } - if err = sl.addReportSample(app, samplesPostRelabelMetric, ts, stale, b); err != nil { + if err = sla.addReportSample(samplesPostRelabelMetric, ts, stale, b, true); err != nil { return err } - if err = sl.addReportSample(app, scrapeSeriesAddedMetric, ts, stale, b); err != nil { + if err = sla.addReportSample(scrapeSeriesAddedMetric, ts, stale, b, true); err != nil { return err } if sl.reportExtraMetrics { - if err = sl.addReportSample(app, scrapeTimeoutMetric, ts, stale, b); err != nil { + if err = sla.addReportSample(scrapeTimeoutMetric, ts, stale, b, true); err != nil { return err } - if err = sl.addReportSample(app, scrapeSampleLimitMetric, ts, stale, b); err != nil { + if err = sla.addReportSample(scrapeSampleLimitMetric, ts, stale, b, true); err != nil { return err } - if err = sl.addReportSample(app, scrapeBodySizeBytesMetric, ts, stale, b); err != nil { + if err = sla.addReportSample(scrapeBodySizeBytesMetric, ts, stale, b, true); err != nil { return err } } return err } -func (sl *scrapeLoop) addReportSample(app storage.Appender, s reportSample, t int64, v float64, b *labels.Builder) error { +func (sl *scrapeLoopAppenderV2) addReportSample(s reportSample, t int64, v float64, b *labels.Builder, rejectOOO bool) error { ce, ok, _ := sl.cache.get(s.name) var ref storage.SeriesRef var lset labels.Labels @@ -2226,21 +2157,19 @@ func (sl *scrapeLoop) addReportSample(app storage.Appender, s reportSample, t in // with scraped metrics in the cache. // We have to drop it when building the actual metric. b.Reset(labels.EmptyLabels()) - b.Set(labels.MetricName, string(s.name[:len(s.name)-1])) + b.Set(model.MetricNameLabel, string(s.name[:len(s.name)-1])) lset = sl.reportSampleMutator(b.Labels()) } - ref, err := app.Append(ref, lset, t, v) + ref, err := sl.Append(ref, lset, 0, t, v, nil, nil, storage.AOptions{ + MetricFamilyName: yoloString(s.name), + Metadata: s.Metadata, + RejectOutOfOrder: rejectOOO, + }) switch { case err == nil: if !ok { sl.cache.addRef(s.name, ref, lset, lset.Hash()) - // We only need to add metadata once a scrape target appears. - if sl.appendMetadataToWAL { - if _, merr := app.UpdateMetadata(ref, lset, s.Metadata); merr != nil { - sl.l.Debug("Error when appending metadata in addReportSample", "ref", fmt.Sprintf("%d", ref), "metadata", fmt.Sprintf("%+v", s.Metadata), "err", merr) - } - } } return nil case errors.Is(err, storage.ErrOutOfOrderSample), errors.Is(err, storage.ErrDuplicateSampleForTimestamp): diff --git a/scrape/scrape_append_v1.go b/scrape/scrape_append_v1.go new file mode 100644 index 0000000000..8a01828ac6 --- /dev/null +++ b/scrape/scrape_append_v1.go @@ -0,0 +1,568 @@ +package scrape + +import ( + "errors" + "fmt" + "io" + "math" + "slices" + "time" + + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/exemplar" + "github.com/prometheus/prometheus/model/histogram" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/model/textparse" + "github.com/prometheus/prometheus/model/timestamp" + "github.com/prometheus/prometheus/model/value" + "github.com/prometheus/prometheus/storage" +) + +// This file contains Appender v1 flow for the temporary compatibility with downstream +// scrape.NewManager users (e.g. OpenTelemetry). +// +// No new changes should be added here. Prometheus do NOT use this code. +// TODO(bwplotka): Remove once Otel has migrated (add issue). + +type scrapeLoopAppenderV1 struct { + *scrapeLoop + + storage.Appender +} + +var _ scrapeLoopAppender = &scrapeLoopAppenderV1{} + +func (sl *scrapeLoop) updateStaleMarkersV1(app storage.Appender, defTime int64) (err error) { + sl.cache.forEachStale(func(ref storage.SeriesRef, lset labels.Labels) bool { + // Series no longer exposed, mark it stale. + app.SetOptions(&storage.AppendOptions{DiscardOutOfOrder: true}) + _, err = app.Append(ref, lset, defTime, math.Float64frombits(value.StaleNaN)) + app.SetOptions(nil) + switch { + case errors.Is(err, storage.ErrOutOfOrderSample), errors.Is(err, storage.ErrDuplicateSampleForTimestamp): + // Do not count these in logging, as this is expected if a target + // goes away and comes back again with a new scrape loop. + err = nil + } + return err == nil + }) + return err +} + +// appenderV1 returns an appender for ingested samples from the target. +func appenderV1(app storage.Appender, sampleLimit, bucketLimit int, maxSchema int32) storage.Appender { + app = &timeLimitAppenderV1{ + Appender: app, + maxTime: timestamp.FromTime(time.Now().Add(maxAheadTime)), + } + + // The sampleLimit is applied after metrics are potentially dropped via relabeling. + if sampleLimit > 0 { + app = &limitAppenderV1{ + Appender: app, + limit: sampleLimit, + } + } + + if bucketLimit > 0 { + app = &bucketLimitAppenderV1{ + Appender: app, + limit: bucketLimit, + } + } + + if maxSchema < histogram.ExponentialSchemaMax { + app = &maxSchemaAppenderV1{ + Appender: app, + maxSchema: maxSchema, + } + } + + return app +} + +func (sl *scrapeLoopAppenderV1) addReportSample(s reportSample, t int64, v float64, b *labels.Builder, rejectOOO bool) error { + ce, ok, _ := sl.cache.get(s.name) + var ref storage.SeriesRef + var lset labels.Labels + if ok { + ref = ce.ref + lset = ce.lset + } else { + // The constants are suffixed with the invalid \xff unicode rune to avoid collisions + // with scraped metrics in the cache. + // We have to drop it when building the actual metric. + b.Reset(labels.EmptyLabels()) + b.Set(model.MetricNameLabel, string(s.name[:len(s.name)-1])) + lset = sl.reportSampleMutator(b.Labels()) + } + + opt := storage.AppendOptions{DiscardOutOfOrder: rejectOOO} + sl.SetOptions(&opt) + ref, err := sl.Append(ref, lset, t, v) + opt.DiscardOutOfOrder = false + sl.SetOptions(&opt) + switch { + case err == nil: + if !ok { + sl.cache.addRef(s.name, ref, lset, lset.Hash()) + // We only need to add metadata once a scrape target appears. + if sl.appendMetadataToWAL { + if _, merr := sl.UpdateMetadata(ref, lset, s.Metadata); merr != nil { + sl.l.Debug("Error when appending metadata in addReportSample", "ref", fmt.Sprintf("%d", ref), "metadata", fmt.Sprintf("%+v", s.Metadata), "err", merr) + } + } + } + return nil + case errors.Is(err, storage.ErrOutOfOrderSample), errors.Is(err, storage.ErrDuplicateSampleForTimestamp): + // Do not log here, as this is expected if a target goes away and comes back + // again with a new scrape loop. + return nil + default: + return err + } +} + +// append for the deprecated storage.Appender flow. +// This is only for downstream project migration purposes and will be removed soon. +func (sl *scrapeLoopAppenderV1) append(b []byte, contentType string, ts time.Time) (total, added, seriesAdded int, err error) { + defTime := timestamp.FromTime(ts) + + if len(b) == 0 { + // Empty scrape. Just update the stale makers and swap the cache (but don't flush it). + err = sl.updateStaleMarkersV1(sl.Appender, defTime) + sl.cache.iterDone(false) + return total, added, seriesAdded, err + } + + p, err := textparse.New(b, contentType, sl.symbolTable, textparse.ParserOptions{ + EnableTypeAndUnitLabels: sl.enableTypeAndUnitLabels, + IgnoreNativeHistograms: !sl.enableNativeHistogramScraping, + ConvertClassicHistogramsToNHCB: sl.convertClassicHistToNHCB, + KeepClassicOnClassicAndNativeHistograms: sl.alwaysScrapeClassicHist, + OpenMetricsSkipSTSeries: sl.enableSTZeroIngestion, + FallbackContentType: sl.fallbackScrapeProtocol, + }) + if p == nil { + sl.l.Error( + "Failed to determine correct type of scrape target.", + "content_type", contentType, + "fallback_media_type", sl.fallbackScrapeProtocol, + "err", err, + ) + return total, added, seriesAdded, err + } + if err != nil { + sl.l.Debug( + "Invalid content type on scrape, using fallback setting.", + "content_type", contentType, + "fallback_media_type", sl.fallbackScrapeProtocol, + "err", err, + ) + } + var ( + appErrs = appendErrors{} + sampleLimitErr error + bucketLimitErr error + lset labels.Labels // Escapes to heap so hoisted out of loop. + e exemplar.Exemplar // Escapes to heap so hoisted out of loop. + lastMeta *metaEntry + lastMFName []byte + ) + + exemplars := make([]exemplar.Exemplar, 0, 1) + + // Take an appender with limits. + app := appenderV1(sl.Appender, sl.sampleLimit, sl.bucketLimit, sl.maxSchema) + + defer func() { + if err != nil { + return + } + // Flush and swap the cache as the scrape was non-empty. + sl.cache.iterDone(true) + }() + +loop: + for { + var ( + et textparse.Entry + sampleAdded, isHistogram bool + met []byte + parsedTimestamp *int64 + val float64 + h *histogram.Histogram + fh *histogram.FloatHistogram + ) + if et, err = p.Next(); err != nil { + if errors.Is(err, io.EOF) { + err = nil + } + break + } + switch et { + // TODO(bwplotka): Consider changing parser to give metadata at once instead of type, help and unit in separation, ideally on `Series()/Histogram() + // otherwise we can expose metadata without series on metadata API. + case textparse.EntryType: + // TODO(bwplotka): Build meta entry directly instead of locking and updating the map. This will + // allow to properly update metadata when e.g unit was added, then removed; + lastMFName, lastMeta = sl.cache.setType(p.Type()) + continue + case textparse.EntryHelp: + lastMFName, lastMeta = sl.cache.setHelp(p.Help()) + continue + case textparse.EntryUnit: + lastMFName, lastMeta = sl.cache.setUnit(p.Unit()) + continue + case textparse.EntryComment: + continue + case textparse.EntryHistogram: + isHistogram = true + default: + } + total++ + + t := defTime + if isHistogram { + met, parsedTimestamp, h, fh = p.Histogram() + } else { + met, parsedTimestamp, val = p.Series() + } + if !sl.honorTimestamps { + parsedTimestamp = nil + } + if parsedTimestamp != nil { + t = *parsedTimestamp + } + + if sl.cache.getDropped(met) { + continue + } + ce, seriesCached, seriesAlreadyScraped := sl.cache.get(met) + var ( + ref storage.SeriesRef + hash uint64 + ) + + if seriesCached { + ref = ce.ref + lset = ce.lset + hash = ce.hash + } else { + p.Labels(&lset) + hash = lset.Hash() + + // Hash label set as it is seen local to the target. Then add target labels + // and relabeling and store the final label set. + lset = sl.sampleMutator(lset) + + // The label set may be set to empty to indicate dropping. + if lset.IsEmpty() { + sl.cache.addDropped(met) + continue + } + + if !lset.Has(labels.MetricName) { + err = errNameLabelMandatory + break loop + } + if !lset.IsValid(sl.validationScheme) { + err = fmt.Errorf("invalid metric name or label names: %s", lset.String()) + break loop + } + + // If any label limits is exceeded the scrape should fail. + if err = verifyLabelLimits(lset, sl.labelLimits); err != nil { + sl.metrics.targetScrapePoolExceededLabelLimits.Inc() + break loop + } + } + + if seriesAlreadyScraped && parsedTimestamp == nil { + err = storage.ErrDuplicateSampleForTimestamp + } else { + if sl.enableSTZeroIngestion { + if stMs := p.StartTimestamp(); stMs != 0 { + if isHistogram { + if h != nil { + ref, err = app.AppendHistogramSTZeroSample(ref, lset, t, stMs, h, nil) + } else { + ref, err = app.AppendHistogramSTZeroSample(ref, lset, t, stMs, nil, fh) + } + } else { + ref, err = app.AppendSTZeroSample(ref, lset, t, stMs) + } + if err != nil && !errors.Is(err, storage.ErrOutOfOrderST) { // OOO is a common case, ignoring completely for now. + // ST is an experimental feature. For now, we don't need to fail the + // scrape on errors updating the created timestamp, log debug. + sl.l.Debug("Error when appending ST in scrape loop", "series", string(met), "ct", stMs, "t", t, "err", err) + } + } + } + + if isHistogram { + if h != nil { + ref, err = app.AppendHistogram(ref, lset, t, h, nil) + } else { + ref, err = app.AppendHistogram(ref, lset, t, nil, fh) + } + } else { + ref, err = app.Append(ref, lset, t, val) + } + } + + if err == nil { + if (parsedTimestamp == nil || sl.trackTimestampsStaleness) && ce != nil { + sl.cache.trackStaleness(ce.ref, ce) + } + } + + // NOTE: exemplars are nil here for v1 appender flow. We append and check the exemplar errors later on. + sampleAdded, err = sl.checkAddError(met, nil, err, &sampleLimitErr, &bucketLimitErr, &appErrs) + if err != nil { + if !errors.Is(err, storage.ErrNotFound) { + sl.l.Debug("Unexpected error", "series", string(met), "err", err) + } + break loop + } + + // If series wasn't cached (is new, not seen on previous scrape) we need need to add it to the scrape cache. + // But we only do this for series that were appended to TSDB without errors. + // If a series was new but we didn't append it due to sample_limit or other errors then we don't need + // it in the scrape cache because we don't need to emit StaleNaNs for it when it disappears. + if !seriesCached && sampleAdded { + ce = sl.cache.addRef(met, ref, lset, hash) + if ce != nil && (parsedTimestamp == nil || sl.trackTimestampsStaleness) { + // Bypass staleness logic if there is an explicit timestamp. + // But make sure we only do this if we have a cache entry (ce) for our series. + sl.cache.trackStaleness(ref, ce) + } + if sampleAdded && sampleLimitErr == nil && bucketLimitErr == nil { + seriesAdded++ + } + } + + // Increment added even if there's an error so we correctly report the + // number of samples remaining after relabeling. + // We still report duplicated samples here since this number should be the exact number + // of time series exposed on a scrape after relabelling. + added++ + exemplars = exemplars[:0] // Reset and reuse the exemplar slice. + for hasExemplar := p.Exemplar(&e); hasExemplar; hasExemplar = p.Exemplar(&e) { + if !e.HasTs { + if isHistogram { + // We drop exemplars for native histograms if they don't have a timestamp. + // Missing timestamps are deliberately not supported as we want to start + // enforcing timestamps for exemplars as otherwise proper deduplication + // is inefficient and purely based on heuristics: we cannot distinguish + // between repeated exemplars and new instances with the same values. + // This is done silently without logs as it is not an error but out of spec. + // This does not affect classic histograms so that behaviour is unchanged. + e = exemplar.Exemplar{} // Reset for next time round loop. + continue + } + e.Ts = t + } + exemplars = append(exemplars, e) + e = exemplar.Exemplar{} // Reset for next time round loop. + } + // Sort so that checking for duplicates / out of order is more efficient during validation. + slices.SortFunc(exemplars, exemplar.Compare) + outOfOrderExemplars := 0 + for _, e := range exemplars { + _, exemplarErr := app.AppendExemplar(ref, lset, e) + switch { + case exemplarErr == nil: + // Do nothing. + case errors.Is(exemplarErr, storage.ErrOutOfOrderExemplar): + outOfOrderExemplars++ + default: + // Since exemplar storage is still experimental, we don't fail the scrape on ingestion errors. + sl.l.Debug("Error while adding exemplar in AddExemplar", "exemplar", fmt.Sprintf("%+v", e), "err", exemplarErr) + } + } + if outOfOrderExemplars > 0 && outOfOrderExemplars == len(exemplars) { + // Only report out of order exemplars if all are out of order, otherwise this was a partial update + // to some existing set of exemplars. + appErrs.numExemplarOutOfOrder += outOfOrderExemplars + sl.l.Debug("Out of order exemplars", "count", outOfOrderExemplars, "latest", fmt.Sprintf("%+v", exemplars[len(exemplars)-1])) + sl.metrics.targetScrapeExemplarOutOfOrder.Add(float64(outOfOrderExemplars)) + } + + if sl.appendMetadataToWAL && lastMeta != nil { + // Is it new series OR did metadata change for this family? + if !seriesCached || lastMeta.lastIterChange == sl.cache.iter { + // In majority cases we can trust that the current series/histogram is matching the lastMeta and lastMFName. + // However, optional TYPE etc metadata and broken OM text can break this, detect those cases here. + // TODO(bwplotka): Consider moving this to parser as many parser users end up doing this (e.g. ST and NHCB parsing). + if isSeriesPartOfFamily(lset.Get(model.MetricNameLabel), lastMFName, lastMeta.Type) { + if _, merr := app.UpdateMetadata(ref, lset, lastMeta.Metadata); merr != nil { + // No need to fail the scrape on errors appending metadata. + sl.l.Debug("Error when appending metadata in scrape loop", "ref", fmt.Sprintf("%d", ref), "metadata", fmt.Sprintf("%+v", lastMeta.Metadata), "err", merr) + } + } + } + } + } + if sampleLimitErr != nil { + if err == nil { + err = sampleLimitErr + } + // We only want to increment this once per scrape, so this is Inc'd outside the loop. + sl.metrics.targetScrapeSampleLimit.Inc() + } + if bucketLimitErr != nil { + if err == nil { + err = bucketLimitErr // If sample limit is hit, that error takes precedence. + } + // We only want to increment this once per scrape, so this is Inc'd outside the loop. + sl.metrics.targetScrapeNativeHistogramBucketLimit.Inc() + } + if appErrs.numOutOfOrder > 0 { + sl.l.Warn("Error on ingesting out-of-order samples", "num_dropped", appErrs.numOutOfOrder) + } + if appErrs.numDuplicates > 0 { + sl.l.Warn("Error on ingesting samples with different value but same timestamp", "num_dropped", appErrs.numDuplicates) + } + if appErrs.numOutOfBounds > 0 { + sl.l.Warn("Error on ingesting samples that are too old or are too far into the future", "num_dropped", appErrs.numOutOfBounds) + } + if appErrs.numExemplarOutOfOrder > 0 { + sl.l.Warn("Error on ingesting out-of-order exemplars", "num_dropped", appErrs.numExemplarOutOfOrder) + } + if err == nil { + err = sl.updateStaleMarkersV1(app, defTime) + } + return total, added, seriesAdded, err +} + +// limitAppenderV1 limits the number of total appended samples in a batch. +type limitAppenderV1 struct { + storage.Appender + + limit int + i int +} + +func (app *limitAppenderV1) Append(ref storage.SeriesRef, lset labels.Labels, t int64, v float64) (storage.SeriesRef, error) { + // Bypass sample_limit checks only if we have a staleness marker for a known series (ref value is non-zero). + // This ensures that if a series is already in TSDB then we always write the marker. + if ref == 0 || !value.IsStaleNaN(v) { + app.i++ + if app.i > app.limit { + return 0, errSampleLimit + } + } + ref, err := app.Appender.Append(ref, lset, t, v) + if err != nil { + return 0, err + } + return ref, nil +} + +func (app *limitAppenderV1) AppendHistogram(ref storage.SeriesRef, lset labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { + // Bypass sample_limit checks only if we have a staleness marker for a known series (ref value is non-zero). + // This ensures that if a series is already in TSDB then we always write the marker. + if ref == 0 || (h != nil && !value.IsStaleNaN(h.Sum)) || (fh != nil && !value.IsStaleNaN(fh.Sum)) { + app.i++ + if app.i > app.limit { + return 0, errSampleLimit + } + } + ref, err := app.Appender.AppendHistogram(ref, lset, t, h, fh) + if err != nil { + return 0, err + } + return ref, nil +} + +type timeLimitAppenderV1 struct { + storage.Appender + + maxTime int64 +} + +func (app *timeLimitAppenderV1) Append(ref storage.SeriesRef, lset labels.Labels, t int64, v float64) (storage.SeriesRef, error) { + if t > app.maxTime { + return 0, storage.ErrOutOfBounds + } + + ref, err := app.Appender.Append(ref, lset, t, v) + if err != nil { + return 0, err + } + return ref, nil +} + +// bucketLimitAppenderV1 limits the number of total appended samples in a batch. +type bucketLimitAppenderV1 struct { + storage.Appender + + limit int +} + +func (app *bucketLimitAppenderV1) AppendHistogram(ref storage.SeriesRef, lset labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { + var err error + if h != nil { + // Return with an early error if the histogram has too many buckets and the + // schema is not exponential, in which case we can't reduce the resolution. + if len(h.PositiveBuckets)+len(h.NegativeBuckets) > app.limit && !histogram.IsExponentialSchema(h.Schema) { + return 0, errBucketLimit + } + for len(h.PositiveBuckets)+len(h.NegativeBuckets) > app.limit { + if h.Schema <= histogram.ExponentialSchemaMin { + return 0, errBucketLimit + } + if err = h.ReduceResolution(h.Schema - 1); err != nil { + return 0, err + } + } + } + if fh != nil { + // Return with an early error if the histogram has too many buckets and the + // schema is not exponential, in which case we can't reduce the resolution. + if len(fh.PositiveBuckets)+len(fh.NegativeBuckets) > app.limit && !histogram.IsExponentialSchema(fh.Schema) { + return 0, errBucketLimit + } + for len(fh.PositiveBuckets)+len(fh.NegativeBuckets) > app.limit { + if fh.Schema <= histogram.ExponentialSchemaMin { + return 0, errBucketLimit + } + if err = fh.ReduceResolution(fh.Schema - 1); err != nil { + return 0, err + } + } + } + if ref, err = app.Appender.AppendHistogram(ref, lset, t, h, fh); err != nil { + return 0, err + } + return ref, nil +} + +type maxSchemaAppenderV1 struct { + storage.Appender + + maxSchema int32 +} + +func (app *maxSchemaAppenderV1) AppendHistogram(ref storage.SeriesRef, lset labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { + var err error + if h != nil { + if histogram.IsExponentialSchemaReserved(h.Schema) && h.Schema > app.maxSchema { + if err = h.ReduceResolution(app.maxSchema); err != nil { + return 0, err + } + } + } + if fh != nil { + if histogram.IsExponentialSchemaReserved(fh.Schema) && fh.Schema > app.maxSchema { + if err = fh.ReduceResolution(app.maxSchema); err != nil { + return 0, err + } + } + } + if ref, err = app.Appender.AppendHistogram(ref, lset, t, h, fh); err != nil { + return 0, err + } + return ref, nil +} diff --git a/scrape/scrape_append_v1_test.go b/scrape/scrape_append_v1_test.go new file mode 100644 index 0000000000..458c704765 --- /dev/null +++ b/scrape/scrape_append_v1_test.go @@ -0,0 +1,1222 @@ +package scrape + +import ( + "context" + + "github.com/prometheus/prometheus/model/exemplar" + "github.com/prometheus/prometheus/model/histogram" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/model/metadata" + "github.com/prometheus/prometheus/storage" +) + +// Collection of tests for Appender v1 flow. +// TODO(bwplotka): Remove once Otel uses v2 flow. + +type nopAppendableV1 struct{} + +func (nopAppendableV1) Appender(context.Context) storage.Appender { + return nopAppenderV1{} +} + +type nopAppenderV1 struct{} + +func (nopAppenderV1) SetOptions(*storage.AppendOptions) {} + +func (nopAppenderV1) Append(storage.SeriesRef, labels.Labels, int64, float64) (storage.SeriesRef, error) { + return 1, nil +} + +func (nopAppenderV1) AppendExemplar(storage.SeriesRef, labels.Labels, exemplar.Exemplar) (storage.SeriesRef, error) { + return 2, nil +} + +func (nopAppenderV1) AppendHistogram(storage.SeriesRef, labels.Labels, int64, *histogram.Histogram, *histogram.FloatHistogram) (storage.SeriesRef, error) { + return 3, nil +} + +func (nopAppenderV1) AppendHistogramSTZeroSample(storage.SeriesRef, labels.Labels, int64, int64, *histogram.Histogram, *histogram.FloatHistogram) (storage.SeriesRef, error) { + return 0, nil +} + +func (nopAppenderV1) UpdateMetadata(storage.SeriesRef, labels.Labels, metadata.Metadata) (storage.SeriesRef, error) { + return 4, nil +} + +func (nopAppenderV1) AppendSTZeroSample(storage.SeriesRef, labels.Labels, int64, int64) (storage.SeriesRef, error) { + return 5, nil +} + +func (nopAppenderV1) Commit() error { return nil } +func (nopAppenderV1) Rollback() error { return nil } + +/* +type collectResultAppendableV1 struct { + *collectResultAppenderV1 +} + +func (a *collectResultAppendableV1) Appender(context.Context) storage.Appender { + return a +} + +// collectResultAppenderV1 records all samples that were added through the appender. +// It can be used as its zero value or be backed by another appender it writes samples through. +type collectResultAppenderV1 struct { + mtx sync.Mutex + + next storage.Appender + resultFloats []floatSample + pendingFloats []floatSample + rolledbackFloats []floatSample + resultHistograms []histogramSample + pendingHistograms []histogramSample + rolledbackHistograms []histogramSample + resultExemplars []exemplar.Exemplar + pendingExemplars []exemplar.Exemplar + resultMetadata []metadataEntry + pendingMetadata []metadataEntry +} + +func (*collectResultAppenderV1) SetOptions(*storage.AppendOptions) {} + +func (a *collectResultAppenderV1) Append(ref storage.SeriesRef, lset labels.Labels, t int64, v float64) (storage.SeriesRef, error) { + a.mtx.Lock() + defer a.mtx.Unlock() + a.pendingFloats = append(a.pendingFloats, floatSample{ + metric: lset, + t: t, + f: v, + }) + + if a.next == nil { + if ref == 0 { + // Use labels hash as a stand-in for unique series reference, to avoid having to track all series. + ref = storage.SeriesRef(lset.Hash()) + } + return ref, nil + } + + ref, err := a.next.Append(ref, lset, t, v) + if err != nil { + return 0, err + } + return ref, nil +} + +func (a *collectResultAppenderV1) AppendExemplar(ref storage.SeriesRef, l labels.Labels, e exemplar.Exemplar) (storage.SeriesRef, error) { + a.mtx.Lock() + defer a.mtx.Unlock() + a.pendingExemplars = append(a.pendingExemplars, e) + if a.next == nil { + return 0, nil + } + + return a.next.AppendExemplar(ref, l, e) +} + +func (a *collectResultAppenderV1) AppendHistogram(ref storage.SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { + a.mtx.Lock() + defer a.mtx.Unlock() + a.pendingHistograms = append(a.pendingHistograms, histogramSample{h: h, fh: fh, t: t, metric: l}) + if a.next == nil { + return 0, nil + } + + return a.next.AppendHistogram(ref, l, t, h, fh) +} + +func (a *collectResultAppenderV1) AppendHistogramSTZeroSample(ref storage.SeriesRef, l labels.Labels, _, st int64, h *histogram.Histogram, _ *histogram.FloatHistogram) (storage.SeriesRef, error) { + if h != nil { + return a.AppendHistogram(ref, l, st, &histogram.Histogram{}, nil) + } + return a.AppendHistogram(ref, l, st, nil, &histogram.FloatHistogram{}) +} + +func (a *collectResultAppenderV1) UpdateMetadata(ref storage.SeriesRef, l labels.Labels, m metadata.Metadata) (storage.SeriesRef, error) { + a.mtx.Lock() + defer a.mtx.Unlock() + a.pendingMetadata = append(a.pendingMetadata, metadataEntry{metric: l, m: m}) + if a.next == nil { + if ref == 0 { + ref = storage.SeriesRef(l.Hash()) + } + return ref, nil + } + + return a.next.UpdateMetadata(ref, l, m) +} + +func (a *collectResultAppenderV1) AppendSTZeroSample(ref storage.SeriesRef, l labels.Labels, _, st int64) (storage.SeriesRef, error) { + return a.Append(ref, l, st, 0.0) +} + +func (a *collectResultAppenderV1) Commit() error { + a.mtx.Lock() + defer a.mtx.Unlock() + a.resultFloats = append(a.resultFloats, a.pendingFloats...) + a.resultExemplars = append(a.resultExemplars, a.pendingExemplars...) + a.resultHistograms = append(a.resultHistograms, a.pendingHistograms...) + a.resultMetadata = append(a.resultMetadata, a.pendingMetadata...) + a.pendingFloats = nil + a.pendingExemplars = nil + a.pendingHistograms = nil + a.pendingMetadata = nil + if a.next == nil { + return nil + } + return a.next.Commit() +} + +func (a *collectResultAppenderV1) Rollback() error { + a.mtx.Lock() + defer a.mtx.Unlock() + a.rolledbackFloats = a.pendingFloats + a.rolledbackHistograms = a.pendingHistograms + a.pendingFloats = nil + a.pendingHistograms = nil + if a.next == nil { + return nil + } + return a.next.Rollback() +} + +func (a *collectResultAppenderV1) String() string { + var sb strings.Builder + for _, s := range a.resultFloats { + sb.WriteString(fmt.Sprintf("committed: %s %f %d\n", s.metric, s.f, s.t)) + } + for _, s := range a.pendingFloats { + sb.WriteString(fmt.Sprintf("pending: %s %f %d\n", s.metric, s.f, s.t)) + } + for _, s := range a.rolledbackFloats { + sb.WriteString(fmt.Sprintf("rolledback: %s %f %d\n", s.metric, s.f, s.t)) + } + return sb.String() +} + +func runManagersV1(t *testing.T, ctx context.Context, opts *Options, app storage.Appendable) (*discovery.Manager, *Manager) { + t.Helper() + + if opts == nil { + opts = &Options{} + } + opts.DiscoveryReloadInterval = model.Duration(100 * time.Millisecond) + if app == nil { + app = nopAppendableV1{} + } + + reg := prometheus.NewRegistry() + sdMetrics, err := discovery.RegisterSDMetrics(reg, discovery.NewRefreshMetrics(reg)) + require.NoError(t, err) + discoveryManager := discovery.NewManager( + ctx, + promslog.NewNopLogger(), + reg, + sdMetrics, + discovery.Updatert(100*time.Millisecond), + ) + scrapeManager, err := NewManager( + opts, + nil, + nil, + app, + prometheus.NewRegistry(), + ) + require.NoError(t, err) + go discoveryManager.Run() + go scrapeManager.Run(discoveryManager.SyncCh()) + return discoveryManager, scrapeManager +} + +// TestManagerSTZeroIngestion_AppenderV1 tests scrape manager for various ST cases. +func TestManagerSTZeroIngestion_AppenderV1(t *testing.T) { + t.Parallel() + const ( + // _total suffix is required, otherwise expfmt with OMText will mark metric as "unknown" + expectedMetricName = "expected_metric_total" + expectedCreatedMetricName = "expected_metric_created" + expectedSampleValue = 17.0 + ) + + for _, testFormat := range []config.ScrapeProtocol{config.PrometheusProto, config.OpenMetricsText1_0_0} { + t.Run(fmt.Sprintf("format=%s", testFormat), func(t *testing.T) { + for _, testWithST := range []bool{false, true} { + t.Run(fmt.Sprintf("withST=%v", testWithST), func(t *testing.T) { + for _, testSTZeroIngest := range []bool{false, true} { + t.Run(fmt.Sprintf("ctZeroIngest=%v", testSTZeroIngest), func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sampleTs := time.Now() + stTs := time.Time{} + if testWithST { + stTs = sampleTs.Add(-2 * time.Minute) + } + + // TODO(bwplotka): Add more types than just counter? + encoded := prepareTestEncodedCounter(t, testFormat, expectedMetricName, expectedSampleValue, sampleTs, stTs) + + app := &collectResultAppenderV1{} + discoveryManager, scrapeManager := runManagersV1(t, ctx, &Options{ + EnableStartTimestampZeroIngestion: testSTZeroIngest, + skipOffsetting: true, + }, &collectResultAppendableV1{app}) + defer scrapeManager.Stop() + + server := setupTestServer(t, config.ScrapeProtocolsHeaders[testFormat], encoded) + serverURL, err := url.Parse(server.URL) + require.NoError(t, err) + + testConfig := fmt.Sprintf(` +global: + # Disable regular scrapes. + scrape_interval: 9999m + scrape_timeout: 5s + +scrape_configs: +- job_name: test + honor_timestamps: true + static_configs: + - targets: ['%s'] +`, serverURL.Host) + applyConfig(t, testConfig, scrapeManager, discoveryManager) + + // Wait for one scrape. + ctx, cancel = context.WithTimeout(ctx, 1*time.Minute) + defer cancel() + require.NoError(t, runutil.Retry(100*time.Millisecond, ctx.Done(), func() error { + app.mtx.Lock() + defer app.mtx.Unlock() + + // Check if scrape happened and grab the relevant samples. + if len(app.resultFloats) > 0 { + return nil + } + return errors.New("expected some float samples, got none") + }), "after 1 minute") + + // Verify results. + // Verify what we got vs expectations around ST injection. + samples := findSamplesForMetric(app.resultFloats, expectedMetricName) + if testWithST && testSTZeroIngest { + require.Len(t, samples, 2) + require.Equal(t, 0.0, samples[0].f) + require.Equal(t, timestamp.FromTime(stTs), samples[0].t) + require.Equal(t, expectedSampleValue, samples[1].f) + require.Equal(t, timestamp.FromTime(sampleTs), samples[1].t) + } else { + require.Len(t, samples, 1) + require.Equal(t, expectedSampleValue, samples[0].f) + require.Equal(t, timestamp.FromTime(sampleTs), samples[0].t) + } + + // Verify what we got vs expectations around additional _created series for OM text. + // enableSTZeroInjection also kills that _created line. + createdSeriesSamples := findSamplesForMetric(app.resultFloats, expectedCreatedMetricName) + if testFormat == config.OpenMetricsText1_0_0 && testWithST && !testSTZeroIngest { + // For OM Text, when counter has ST, and feature flag disabled we should see _created lines. + require.Len(t, createdSeriesSamples, 1) + // Conversion taken from common/expfmt.writeOpenMetricsFloat. + // We don't check the st timestamp as explicit ts was not implemented in expfmt.Encoder, + // but exists in OM https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#:~:text=An%20example%20with%20a%20Metric%20with%20no%20labels%2C%20and%20a%20MetricPoint%20with%20a%20timestamp%20and%20a%20created + // We can implement this, but we want to potentially get rid of OM 1.0 ST lines + require.Equal(t, float64(timestamppb.New(stTs).AsTime().UnixNano())/1e9, createdSeriesSamples[0].f) + } else { + require.Empty(t, createdSeriesSamples) + } + }) + } + }) + } + }) + } +} + +func TestManagerSTZeroIngestionHistogram_AppenderV1(t *testing.T) { + t.Parallel() + const mName = "expected_histogram" + + for _, tc := range []struct { + name string + inputHistSample *dto.Histogram + enableSTZeroIngestion bool + }{ + { + name: "disabled with ST on histogram", + inputHistSample: func() *dto.Histogram { + h := generateTestHistogram(0) + h.CreatedTimestamp = timestamppb.Now() + return h + }(), + enableSTZeroIngestion: false, + }, + { + name: "enabled with ST on histogram", + inputHistSample: func() *dto.Histogram { + h := generateTestHistogram(0) + h.CreatedTimestamp = timestamppb.Now() + return h + }(), + enableSTZeroIngestion: true, + }, + { + name: "enabled without ST on histogram", + inputHistSample: func() *dto.Histogram { + h := generateTestHistogram(0) + return h + }(), + enableSTZeroIngestion: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + app := &collectResultAppenderV1{} + discoveryManager, scrapeManager := runManagersV1(t, ctx, &Options{ + EnableStartTimestampZeroIngestion: tc.enableSTZeroIngestion, + skipOffsetting: true, + }, &collectResultAppendableV1{app}) + defer scrapeManager.Stop() + + once := sync.Once{} + // Start fake HTTP target to that allow one scrape only. + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + fail := true + once.Do(func() { + fail = false + w.Header().Set("Content-Type", `application/vnd.google.protobuf; proto=io.prometheus.client.MetricFamily; encoding=delimited`) + + ctrType := dto.MetricType_HISTOGRAM + w.Write(protoMarshalDelimited(t, &dto.MetricFamily{ + Name: proto.String(mName), + Type: &ctrType, + Metric: []*dto.Metric{{Histogram: tc.inputHistSample}}, + })) + }) + + if fail { + w.WriteHeader(http.StatusInternalServerError) + } + }), + ) + defer server.Close() + + serverURL, err := url.Parse(server.URL) + require.NoError(t, err) + + testConfig := fmt.Sprintf(` +global: + # Disable regular scrapes. + scrape_interval: 9999m + scrape_timeout: 5s + +scrape_configs: +- job_name: test + scrape_native_histograms: true + static_configs: + - targets: ['%s'] +`, serverURL.Host) + applyConfig(t, testConfig, scrapeManager, discoveryManager) + + var got []histogramSample + + // Wait for one scrape. + ctx, cancel = context.WithTimeout(ctx, 1*time.Minute) + defer cancel() + require.NoError(t, runutil.Retry(100*time.Millisecond, ctx.Done(), func() error { + app.mtx.Lock() + defer app.mtx.Unlock() + + // Check if scrape happened and grab the relevant histograms, they have to be there - or it's a bug + // and it's not worth waiting. + for _, h := range app.resultHistograms { + if h.metric.Get(model.MetricNameLabel) == mName { + got = append(got, h) + } + } + if len(app.resultHistograms) > 0 { + return nil + } + return errors.New("expected some histogram samples, got none") + }), "after 1 minute") + + // Check for zero samples, assuming we only injected always one histogram sample. + // Did it contain ST to inject? If yes, was ST zero enabled? + if tc.inputHistSample.CreatedTimestamp.IsValid() && tc.enableSTZeroIngestion { + require.Len(t, got, 2) + // Zero sample. + require.Equal(t, histogram.Histogram{}, *got[0].h) + // Quick soft check to make sure it's the same sample or at least not zero. + require.Equal(t, tc.inputHistSample.GetSampleSum(), got[1].h.Sum) + return + } + + // Expect only one, valid sample. + require.Len(t, got, 1) + // Quick soft check to make sure it's the same sample or at least not zero. + require.Equal(t, tc.inputHistSample.GetSampleSum(), got[0].h.Sum) + }) + } +} + +// TestNHCBAndSTZeroIngestion_AppenderV1 verifies that both ConvertClassicHistogramsToNHCBEnabled +// and EnableStartTimestampZeroIngestion can be used simultaneously without errors. +// This test addresses issue #17216 by ensuring the previously blocking check has been removed. +// The test verifies that the presence of exemplars in the input does not cause errors, +// although exemplars are not preserved during NHCB conversion (as documented below). +func TestNHCBAndSTZeroIngestion_AppenderV1(t *testing.T) { + t.Parallel() + + const ( + mName = "test_histogram" + // The expected sum of the histogram, as defined by the test's OpenMetrics exposition data. + // This value (45.5) is the sum reported in the test_histogram_sum metric below. + expectedHistogramSum = 45.5 + ) + + ctx := t.Context() + + app := &collectResultAppenderV1{} + discoveryManager, scrapeManager := runManagersV1(t, ctx, &Options{ + EnableStartTimestampZeroIngestion: true, + skipOffsetting: true, + }, &collectResultAppendableV1{app}) + defer scrapeManager.Stop() + + once := sync.Once{} + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + fail := true + once.Do(func() { + fail = false + w.Header().Set("Content-Type", `application/openmetrics-text`) + + // Expose a histogram with created timestamp and exemplars to verify no parsing errors occur. + fmt.Fprint(w, `# HELP test_histogram A histogram with created timestamp and exemplars +# TYPE test_histogram histogram +test_histogram_bucket{le="0.0"} 1 +test_histogram_bucket{le="1.0"} 10 # {trace_id="trace-1"} 0.5 123456789 +test_histogram_bucket{le="2.0"} 20 # {trace_id="trace-2"} 1.5 123456780 +test_histogram_bucket{le="+Inf"} 30 # {trace_id="trace-3"} 2.5 +test_histogram_count 30 +test_histogram_sum 45.5 +test_histogram_created 1520430001 +# EOF +`) + }) + + if fail { + w.WriteHeader(http.StatusInternalServerError) + } + }), + ) + defer server.Close() + + serverURL, err := url.Parse(server.URL) + require.NoError(t, err) + + // Configuration with both convert_classic_histograms_to_nhcb enabled and ST zero ingestion enabled. + testConfig := fmt.Sprintf(` +global: + # Use a very long scrape_interval to prevent automatic scraping during the test. + scrape_interval: 9999m + scrape_timeout: 5s + +scrape_configs: +- job_name: test + convert_classic_histograms_to_nhcb: true + static_configs: + - targets: ['%s'] +`, serverURL.Host) + + applyConfig(t, testConfig, scrapeManager, discoveryManager) + + // Verify that the scrape pool was created (proves the blocking check was removed). + require.Eventually(t, func() bool { + scrapeManager.mtxScrape.Lock() + defer scrapeManager.mtxScrape.Unlock() + _, exists := scrapeManager.scrapePools["test"] + return exists + }, 5*time.Second, 100*time.Millisecond, "scrape pool should be created for job 'test'") + + // Helper function to get matching histograms to avoid race conditions. + getMatchingHistograms := func() []histogramSample { + app.mtx.Lock() + defer app.mtx.Unlock() + + var got []histogramSample + for _, h := range app.resultHistograms { + if h.metric.Get(model.MetricNameLabel) == mName { + got = append(got, h) + } + } + return got + } + + require.Eventually(t, func() bool { + return len(getMatchingHistograms()) > 0 + }, 1*time.Minute, 100*time.Millisecond, "expected histogram samples, got none") + + // Verify that samples were ingested (proving both features work together). + got := getMatchingHistograms() + + // With ST zero ingestion enabled and a created timestamp present, we expect 2 samples: + // one zero sample and one actual sample. + require.Len(t, got, 2, "expected 2 histogram samples (zero sample + actual sample)") + require.Equal(t, histogram.Histogram{}, *got[0].h, "first sample should be zero sample") + require.InDelta(t, expectedHistogramSum, got[1].h.Sum, 1e-9, "second sample should retain the expected sum") + require.Len(t, app.resultExemplars, 2, "expected 2 exemplars from histogram buckets") +} + +func TestManagerDisableEndOfRunStalenessMarkers_AppenderV1(t *testing.T) { + cfgText := ` +scrape_configs: + - job_name: one + scrape_interval: 1m + scrape_timeout: 1m + - job_name: two + scrape_interval: 1m + scrape_timeout: 1m +` + + cfg := loadConfiguration(t, cfgText) + + m, err := NewManager(&Options{}, nil, nil, &nopAppendableV1{}, prometheus.NewRegistry()) + require.NoError(t, err) + defer m.Stop() + require.NoError(t, m.ApplyConfig(cfg)) + + // Pass targets to the manager. + tgs := map[string][]*targetgroup.Group{ + "one": {{Targets: []model.LabelSet{{"__address__": "h1"}, {"__address__": "h2"}, {"__address__": "h3"}}}}, + "two": {{Targets: []model.LabelSet{{"__address__": "h4"}}}}, + } + m.updateTsets(tgs) + m.reload() + + activeTargets := m.TargetsActive() + targetsToDisable := []*Target{ + activeTargets["one"][0], + activeTargets["one"][2], + } + + // Disable end of run staleness markers for some targets. + m.DisableEndOfRunStalenessMarkers("one", targetsToDisable) + // This should be a no-op + m.DisableEndOfRunStalenessMarkers("non-existent-job", targetsToDisable) + + // Check that the end of run staleness markers are disabled for the correct targets. + for _, group := range []string{"one", "two"} { + for _, tg := range activeTargets[group] { + loop := m.scrapePools[group].loops[tg.hash()].(*scrapeLoop) + expectedDisabled := slices.Contains(targetsToDisable, tg) + require.Equal(t, expectedDisabled, loop.disabledEndOfRunStalenessMarkers.Load()) + } + } +} + +// maybeLimitedIsEnoughAppendableV1 panics if anything other than appending sample is invoked +// TODO(bwplotka): Move to storage interface_compact if applicable wider (and error out?) +type maybeLimitedIsEnoughAppendableV1 struct { + app *storage.LimitedAppenderV1 +} + +func (a *maybeLimitedIsEnoughAppendableV1) Appender(ctx context.Context) storage.Appender { + return maybeLimitedV1IsEnoughAppenderV1{a: a.app} +} + +type maybeLimitedV1IsEnoughAppenderV1 struct { + storage.Appender + a *storage.LimitedAppenderV1 +} + +func (a *maybeLimitedV1IsEnoughAppenderV1) Append(ref storage.SeriesRef, l labels.Labels, t int64, v float64) (storage.SeriesRef, error) { + return a.a.Append(ref, l, t, v) +} + +func (a *maybeLimitedV1IsEnoughAppenderV1) AppendHistogram(ref storage.SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { + return a.a.AppendHistogram(ref, l, t, h, fh) +} + +func (a *maybeLimitedV1IsEnoughAppenderV1) Commit() error { + return a.a.Commit() +} + +func (a *maybeLimitedV1IsEnoughAppenderV1) Rollback() error { + return a.a.Rollback() +} + +func runScrapeLoopTestV1(t *testing.T, s *teststorage.TestStorage, expectOutOfOrder bool) { + // Create an appender for adding samples to the storage. + app := s.Appender(context.Background()) + capp := &collectResultAppenderV1{next: &maybeLimitedV1IsEnoughAppenderV1{a: app}} + sl := newBasicScrapeLoop(t, context.Background(), nil, &collectResultAppendableV1{capp}, nil, 0) + + // Current time for generating timestamps. + now := time.Now() + + // Calculate timestamps for the samples based on the current time. + now = now.Truncate(time.Minute) // round down the now timestamp to the nearest minute + timestampInorder1 := now + timestampOutOfOrder := now.Add(-5 * time.Minute) + timestampInorder2 := now.Add(5 * time.Minute) + + slApp := sl.appender(context.Background()) + _, _, _, err := slApp.append([]byte(`metric_total{a="1",b="1"} 1`), "text/plain", timestampInorder1) + require.NoError(t, err) + + _, _, _, err = slApp.append([]byte(`metric_total{a="1",b="1"} 2`), "text/plain", timestampOutOfOrder) + require.NoError(t, err) + + _, _, _, err = slApp.append([]byte(`metric_total{a="1",b="1"} 3`), "text/plain", timestampInorder2) + require.NoError(t, err) + + require.NoError(t, slApp.Commit()) + + // Query the samples back from the storage. + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + q, err := s.Querier(time.Time{}.UnixNano(), time.Now().UnixNano()) + require.NoError(t, err) + defer q.Close() + + // Use a matcher to filter the metric name. + series := q.Select(ctx, false, nil, labels.MustNewMatcher(labels.MatchRegexp, "__name__", "metric_total")) + + var results []floatSample + for series.Next() { + it := series.At().Iterator(nil) + for it.Next() == chunkenc.ValFloat { + t, v := it.At() + results = append(results, floatSample{ + metric: series.At().Labels(), + t: t, + f: v, + }) + } + require.NoError(t, it.Err()) + } + require.NoError(t, series.Err()) + + // Define the expected results + want := []floatSample{ + { + metric: labels.FromStrings("__name__", "metric_total", "a", "1", "b", "1"), + t: timestamp.FromTime(timestampInorder1), + f: 1, + }, + { + metric: labels.FromStrings("__name__", "metric_total", "a", "1", "b", "1"), + t: timestamp.FromTime(timestampInorder2), + f: 3, + }, + } + + if expectOutOfOrder { + require.NotEqual(t, want, results, "Expected results to include out-of-order sample:\n%s", results) + } else { + require.Equal(t, want, results, "Appended samples not as expected:\n%s", results) + } +} + +// Regression test against https://github.com/prometheus/prometheus/issues/15831. +func TestScrapeAppendMetadataUpdate_AppenderV1(t *testing.T) { + const ( + scrape1 = `# TYPE test_metric counter +# HELP test_metric some help text +# UNIT test_metric metric +test_metric_total 1 +# TYPE test_metric2 gauge +# HELP test_metric2 other help text +test_metric2{foo="bar"} 2 +# TYPE test_metric3 gauge +# HELP test_metric3 this represents tricky case of "broken" text that is not trivial to detect +test_metric3_metric4{foo="bar"} 2 +# EOF` + scrape2 = `# TYPE test_metric counter +# HELP test_metric different help text +test_metric_total 11 +# TYPE test_metric2 gauge +# HELP test_metric2 other help text +# UNIT test_metric2 metric2 +test_metric2{foo="bar"} 22 +# EOF` + ) + + // Create an appender for adding samples to the storage. + capp := &collectResultAppenderV1{} + sl := newBasicScrapeLoop(t, context.Background(), nil, &collectResultAppendableV1{capp}, nil, 0) + + now := time.Now() + slApp := sl.appender(context.Background()) + _, _, _, err := slApp.append([]byte(scrape1), "application/openmetrics-text", now) + require.NoError(t, err) + require.NoError(t, slApp.Commit()) + testutil.RequireEqualWithOptions(t, []metadataEntry{ + {metric: labels.FromStrings("__name__", "test_metric_total"), m: metadata.Metadata{Type: "counter", Unit: "metric", Help: "some help text"}}, + {metric: labels.FromStrings("__name__", "test_metric2", "foo", "bar"), m: metadata.Metadata{Type: "gauge", Unit: "", Help: "other help text"}}, + }, capp.resultMetadata, []cmp.Option{cmp.Comparer(metadataEntryEqual)}) + capp.resultMetadata = nil + + // Next (the same) scrape should not add new metadata entries. + slApp = sl.appender(context.Background()) + _, _, _, err = slApp.append([]byte(scrape1), "application/openmetrics-text", now.Add(15*time.Second)) + require.NoError(t, err) + require.NoError(t, slApp.Commit()) + testutil.RequireEqualWithOptions(t, []metadataEntry(nil), capp.resultMetadata, []cmp.Option{cmp.Comparer(metadataEntryEqual)}) + + slApp = sl.appender(context.Background()) + _, _, _, err = slApp.append([]byte(scrape2), "application/openmetrics-text", now.Add(15*time.Second)) + require.NoError(t, err) + require.NoError(t, slApp.Commit()) + testutil.RequireEqualWithOptions(t, []metadataEntry{ + {metric: labels.FromStrings("__name__", "test_metric_total"), m: metadata.Metadata{Type: "counter", Unit: "metric", Help: "different help text"}}, // Here, technically we should have no unit, but it's a known limitation of the current implementation. + {metric: labels.FromStrings("__name__", "test_metric2", "foo", "bar"), m: metadata.Metadata{Type: "gauge", Unit: "metric2", Help: "other help text"}}, + }, capp.resultMetadata, []cmp.Option{cmp.Comparer(metadataEntryEqual)}) +} + +func TestScrapeReportMetadataUpdate_AppenderV1(t *testing.T) { + // Create an appender for adding samples to the storage. + capp := &collectResultAppenderV1{} + sl := newBasicScrapeLoop(t, context.Background(), nopScraper{}, &collectResultAppendableV1{capp}, nil, 0) + now := time.Now() + slApp := sl.appender(context.Background()) + + require.NoError(t, sl.report(slApp, now, 2*time.Second, 1, 1, 1, 512, nil)) + require.NoError(t, slApp.Commit()) + testutil.RequireEqualWithOptions(t, []metadataEntry{ + {metric: labels.FromStrings("__name__", "up"), m: scrapeHealthMetric.Metadata}, + {metric: labels.FromStrings("__name__", "scrape_duration_seconds"), m: scrapeDurationMetric.Metadata}, + {metric: labels.FromStrings("__name__", "scrape_samples_scraped"), m: scrapeSamplesMetric.Metadata}, + {metric: labels.FromStrings("__name__", "scrape_samples_post_metric_relabeling"), m: samplesPostRelabelMetric.Metadata}, + {metric: labels.FromStrings("__name__", "scrape_series_added"), m: scrapeSeriesAddedMetric.Metadata}, + }, capp.resultMetadata, []cmp.Option{cmp.Comparer(metadataEntryEqual)}) +} + +func TestScrapePoolAppender_AppenderV1(t *testing.T) { + cfg := &config.ScrapeConfig{ + MetricNameValidationScheme: model.UTF8Validation, + MetricNameEscapingScheme: model.AllowUTF8, + } + app := &nopAppendableV1{} + sp, _ := newScrapePool(cfg, app, nil, 0, nil, nil, &Options{}, newTestScrapeMetrics(t)) + + loop := sp.newLoop(scrapeLoopOptions{ + target: &Target{}, + }) + appl, ok := loop.(*scrapeLoop) + require.True(t, ok, "Expected scrapeLoop but got %T", loop) + + wrapped := appenderV1(appl.appendableV1.Appender(context.Background()), 0, 0, histogram.ExponentialSchemaMax) + + tl, ok := wrapped.(*timeLimitAppender) + require.True(t, ok, "Expected timeLimitAppender but got %T", wrapped) + + _, ok = tl.Appender.(nopAppenderV1) + require.True(t, ok, "Expected base appender but got %T", tl.Appender) + + sampleLimit := 100 + loop = sp.newLoop(scrapeLoopOptions{ + target: &Target{}, + sampleLimit: sampleLimit, + }) + appl, ok = loop.(*scrapeLoop) + require.True(t, ok, "Expected scrapeLoop but got %T", loop) + + wrapped = appenderV1(appl.appendableV1.Appender(context.Background()), sampleLimit, 0, histogram.ExponentialSchemaMax) + + sl, ok := wrapped.(*limitAppender) + require.True(t, ok, "Expected limitAppender but got %T", wrapped) + + tl, ok = sl.Appender.(*timeLimitAppender) + require.True(t, ok, "Expected timeLimitAppender but got %T", sl.Appender) + + _, ok = tl.Appender.(nopAppenderV1) + require.True(t, ok, "Expected base appender but got %T", tl.Appender) + + wrapped = appenderV1(appl.appendableV1.Appender(context.Background()), sampleLimit, 100, histogram.ExponentialSchemaMax) + + bl, ok := wrapped.(*bucketLimitAppender) + require.True(t, ok, "Expected bucketLimitAppender but got %T", wrapped) + + sl, ok = bl.Appender.(*limitAppender) + require.True(t, ok, "Expected limitAppender but got %T", bl) + + tl, ok = sl.Appender.(*timeLimitAppender) + require.True(t, ok, "Expected timeLimitAppender but got %T", sl.Appender) + + _, ok = tl.Appender.(nopAppenderV1) + require.True(t, ok, "Expected base appender but got %T", tl.Appender) + + wrapped = appenderV1(appl.appendableV1.Appender(context.Background()), sampleLimit, 100, 0) + + ml, ok := wrapped.(*maxSchemaAppender) + require.True(t, ok, "Expected maxSchemaAppender but got %T", wrapped) + + bl, ok = ml.Appender.(*bucketLimitAppender) + require.True(t, ok, "Expected bucketLimitAppender but got %T", wrapped) + + sl, ok = bl.Appender.(*limitAppender) + require.True(t, ok, "Expected limitAppender but got %T", bl) + + tl, ok = sl.Appender.(*timeLimitAppender) + require.True(t, ok, "Expected timeLimitAppender but got %T", sl.Appender) + + _, ok = tl.Appender.(nopAppenderV1) + require.True(t, ok, "Expected base appender but got %T", tl.Appender) +} + +func TestScrapeLoopStop_AppenderV1(t *testing.T) { + var ( + signal = make(chan struct{}, 1) + scraper = &testScraper{} + ) + + // Since we're writing samples directly below we need to provide a protocol fallback. + capp := &collectResultAppenderV1{} + sl := newBasicScrapeLoopWithFallback(t, context.Background(), scraper, &collectResultAppendableV1{capp}, nil, 10*time.Millisecond, "text/plain") + + // Terminate loop after 2 scrapes. + numScrapes := 0 + + scraper.scrapeFunc = func(ctx context.Context, w io.Writer) error { + numScrapes++ + if numScrapes == 2 { + go sl.stop() + <-sl.ctx.Done() + } + w.Write([]byte("metric_a 42\n")) + return ctx.Err() + } + + go func() { + sl.run(nil) + signal <- struct{}{} + }() + + select { + case <-signal: + case <-time.After(5 * time.Second): + require.FailNow(t, "Scrape wasn't stopped.") + } + + // We expected 1 actual sample for each scrape plus 5 for report samples. + // At least 2 scrapes were made, plus the final stale markers. + require.GreaterOrEqual(t, len(capp.resultFloats), 6*3, "Expected at least 3 scrapes with 6 samples each.") + require.Zero(t, len(capp.resultFloats)%6, "There is a scrape with missing samples.") + // All samples in a scrape must have the same timestamp. + var ts int64 + for i, s := range capp.resultFloats { + switch { + case i%6 == 0: + ts = s.t + case s.t != ts: + t.Fatalf("Unexpected multiple timestamps within single scrape") + } + } + // All samples from the last scrape must be stale markers. + for _, s := range capp.resultFloats[len(capp.resultFloats)-5:] { + require.True(t, value.IsStaleNaN(s.f), "Appended last sample not as expected. Wanted: stale NaN Got: %x", math.Float64bits(s.f)) + } +} + +func TestScrapeLoopRun_AppenderV1(t *testing.T) { + t.Parallel() + var ( + signal = make(chan struct{}, 1) + errc = make(chan error) + + scraper = &testScraper{} + scrapeMetrics = newTestScrapeMetrics(t) + ) + + ctx, cancel := context.WithCancel(context.Background()) + sl := newScrapeLoop(ctx, + scraper, + nil, nil, + nopMutator, + nopMutator, + nopAppendableV1{}, + nil, + nil, + nil, + 0, + true, + false, + true, + 0, 0, histogram.ExponentialSchemaMax, + nil, + time.Second, + time.Hour, + false, + false, + false, + false, + false, + false, + false, + nil, + false, + scrapeMetrics, + false, + model.UTF8Validation, + model.NoEscaping, + "", + ) + + // The loop must terminate during the initial offset if the context + // is canceled. + scraper.offsetDur = time.Hour + + go func() { + sl.run(errc) + signal <- struct{}{} + }() + + // Wait to make sure we are actually waiting on the offset. + time.Sleep(1 * time.Second) + + cancel() + select { + case <-signal: + case <-time.After(5 * time.Second): + require.FailNow(t, "Cancellation during initial offset failed.") + case err := <-errc: + require.FailNow(t, "Unexpected error", "err: %s", err) + } + + // The provided timeout must cause cancellation of the context passed down to the + // scraper. The scraper has to respect the context. + scraper.offsetDur = 0 + + block := make(chan struct{}) + scraper.scrapeFunc = func(ctx context.Context, _ io.Writer) error { + select { + case <-block: + case <-ctx.Done(): + return ctx.Err() + } + return nil + } + + ctx, cancel = context.WithCancel(context.Background()) + sl = newBasicScrapeLoop(t, ctx, scraper, nopAppendableV1{}, nil, time.Second) + sl.timeout = 100 * time.Millisecond + + go func() { + sl.run(errc) + signal <- struct{}{} + }() + + select { + case err := <-errc: + require.ErrorIs(t, err, context.DeadlineExceeded) + case <-time.After(3 * time.Second): + require.FailNow(t, "Expected timeout error but got none.") + } + + // We already caught the timeout error and are certainly in the loop. + // Let the scrapes returns immediately to cause no further timeout errors + // and check whether canceling the parent context terminates the loop. + close(block) + cancel() + + select { + case <-signal: + // Loop terminated as expected. + case err := <-errc: + require.FailNow(t, "Unexpected error", "err: %s", err) + case <-time.After(3 * time.Second): + require.FailNow(t, "Loop did not terminate on context cancellation") + } +} + +func TestScrapeLoopForcedErr_AppenderV1(t *testing.T) { + var ( + signal = make(chan struct{}, 1) + errc = make(chan error) + + scraper = &testScraper{} + ) + + ctx, cancel := context.WithCancel(context.Background()) + sl := newBasicScrapeLoop(t, ctx, scraper, nopAppendableV1{}, nil, time.Second) + + forcedErr := errors.New("forced err") + sl.setForcedError(forcedErr) + + scraper.scrapeFunc = func(context.Context, io.Writer) error { + require.FailNow(t, "Should not be scraped.") + return nil + } + + go func() { + sl.run(errc) + signal <- struct{}{} + }() + + select { + case err := <-errc: + require.ErrorIs(t, err, forcedErr) + case <-time.After(3 * time.Second): + require.FailNow(t, "Expected forced error but got none.") + } + cancel() + + select { + case <-signal: + case <-time.After(5 * time.Second): + require.FailNow(t, "Scrape not stopped.") + } +} + +func TestScrapeLoopMetadata_AppenderV1(t *testing.T) { + var ( + signal = make(chan struct{}) + scraper = &testScraper{} + scrapeMetrics = newTestScrapeMetrics(t) + cache = newScrapeCache(scrapeMetrics) + ) + defer close(signal) + + ctx, cancel := context.WithCancel(context.Background()) + sl := newScrapeLoop(ctx, + scraper, + nil, nil, + nopMutator, + nopMutator, + nopAppendableV1{}, + nil, + cache, + labels.NewSymbolTable(), + 0, + true, + false, + true, + 0, 0, histogram.ExponentialSchemaMax, + nil, + 0, + 0, + false, + false, + false, + false, + false, + false, + false, + nil, + false, + scrapeMetrics, + false, + model.UTF8Validation, + model.NoEscaping, + "", + ) + defer cancel() + + slApp := sl.appender(ctx) + total, _, _, err := slApp.append([]byte(`# TYPE test_metric counter +# HELP test_metric some help text +# UNIT test_metric metric +test_metric_total 1 +# TYPE test_metric_no_help gauge +# HELP test_metric_no_type other help text +# EOF`), "application/openmetrics-text", time.Now()) + require.NoError(t, err) + require.NoError(t, slApp.Commit()) + require.Equal(t, 1, total) + + md, ok := cache.GetMetadata("test_metric") + require.True(t, ok, "expected metadata to be present") + require.Equal(t, model.MetricTypeCounter, md.Type, "unexpected metric type") + require.Equal(t, "some help text", md.Help) + require.Equal(t, "metric", md.Unit) + + md, ok = cache.GetMetadata("test_metric_no_help") + require.True(t, ok, "expected metadata to be present") + require.Equal(t, model.MetricTypeGauge, md.Type, "unexpected metric type") + require.Empty(t, md.Help) + require.Empty(t, md.Unit) + + md, ok = cache.GetMetadata("test_metric_no_type") + require.True(t, ok, "expected metadata to be present") + require.Equal(t, model.MetricTypeUnknown, md.Type, "unexpected metric type") + require.Equal(t, "other help text", md.Help) + require.Empty(t, md.Unit) +} + +func TestScrapeLoopSeriesAdded_AppenderV1(t *testing.T) { + ctx, sl := simpleTestScrapeLoopV1(t) + + slApp := sl.appender(ctx) + total, added, seriesAdded, err := slApp.append([]byte("test_metric 1\n"), "text/plain", time.Time{}) + require.NoError(t, err) + require.NoError(t, slApp.Commit()) + require.Equal(t, 1, total) + require.Equal(t, 1, added) + require.Equal(t, 1, seriesAdded) + + slApp = sl.appender(ctx) + total, added, seriesAdded, err = slApp.append([]byte("test_metric 1\n"), "text/plain", time.Time{}) + require.NoError(t, slApp.Commit()) + require.NoError(t, err) + require.Equal(t, 1, total) + require.Equal(t, 1, added) + require.Equal(t, 0, seriesAdded) +} + +func TestScrapeLoopFailWithInvalidLabelsAfterRelabel_AppenderV1(t *testing.T) { + ctx := t.Context() + + target := &Target{ + labels: labels.FromStrings("pod_label_invalid_012\xff", "test"), + } + relabelConfig := []*relabel.Config{{ + Action: relabel.LabelMap, + Regex: relabel.MustNewRegexp("pod_label_invalid_(.+)"), + Separator: ";", + Replacement: "$1", + NameValidationScheme: model.UTF8Validation, + }} + ctx, sl := simpleTestScrapeLoopV1(t) + sl.sampleMutator = func(l labels.Labels) labels.Labels { + return mutateSampleLabels(l, target, true, relabelConfig) + } + + slApp := sl.appender(ctx) + total, added, seriesAdded, err := slApp.append([]byte("test_metric 1\n"), "text/plain", time.Time{}) + require.ErrorContains(t, err, "invalid metric name or label names") + require.NoError(t, slApp.Rollback()) + require.Equal(t, 1, total) + require.Equal(t, 0, added) + require.Equal(t, 0, seriesAdded) +} + +func TestScrapeLoopFailLegacyUnderUTF8_AppenderV1(t *testing.T) { + ctx := t.Context() + + ctx, sl := simpleTestScrapeLoopV1(t) + sl.validationScheme = model.LegacyValidation + + slApp := sl.appender(ctx) + total, added, seriesAdded, err := slApp.append([]byte("{\"test.metric\"} 1\n"), "text/plain", time.Time{}) + require.ErrorContains(t, err, "invalid metric name or label names") + require.NoError(t, slApp.Rollback()) + require.Equal(t, 1, total) + require.Equal(t, 0, added) + require.Equal(t, 0, seriesAdded) + + // When scrapeloop has validation set to UTF-8, the metric is allowed. + sl.validationScheme = model.UTF8Validation + + slApp = sl.appender(ctx) + total, added, seriesAdded, err = slApp.append([]byte("{\"test.metric\"} 1\n"), "text/plain", time.Time{}) + require.NoError(t, err) + require.Equal(t, 1, total) + require.Equal(t, 1, added) + require.Equal(t, 1, seriesAdded) +} +*/ diff --git a/scrape/scrape_test.go b/scrape/scrape_test.go index 5ccdb80019..9feab7152f 100644 --- a/scrape/scrape_test.go +++ b/scrape/scrape_test.go @@ -70,6 +70,8 @@ import ( "github.com/prometheus/prometheus/util/testutil" ) +type sample = teststorage.Sample + func TestMain(m *testing.M) { testutil.TolerantVerifyLeak(m) } @@ -88,18 +90,25 @@ func newTestScrapeMetrics(t testing.TB) *scrapeMetrics { func TestNewScrapePool(t *testing.T) { var ( - app = &nopAppendable{} - cfg = &config.ScrapeConfig{ + appV1 = &nopAppendableV1{} + appV2 = &nopAppendable{} + cfg = &config.ScrapeConfig{ MetricNameValidationScheme: model.UTF8Validation, MetricNameEscapingScheme: model.AllowUTF8, } - sp, err = newScrapePool(cfg, app, 0, nil, nil, &Options{}, newTestScrapeMetrics(t)) + sp, err = newScrapePool(cfg, appV1, appV2, 0, nil, nil, &Options{}, newTestScrapeMetrics(t)) ) require.NoError(t, err) - a, ok := sp.appendable.(*nopAppendable) + a, ok := sp.appendableV1.(*nopAppendableV1) require.True(t, ok, "Failure to append.") - require.Equal(t, app, a, "Wrong sample appender.") + require.Equal(t, appV1, a, "Wrong sample appender.") + require.Equal(t, cfg, sp.config, "Wrong scrape config.") + require.NotNil(t, sp.newLoop, "newLoop function not initialized.") + + a2, ok := sp.appendableV2.(*nopAppendable) + require.True(t, ok, "Failure to append.") + require.Equal(t, appV2, a2, "Wrong sample appender.") require.Equal(t, cfg, sp.config, "Wrong scrape config.") require.NotNil(t, sp.newLoop, "newLoop function not initialized.") } @@ -108,9 +117,7 @@ func TestStorageHandlesOutOfOrderTimestamps(t *testing.T) { // Test with default OutOfOrderTimeWindow (0) t.Run("Out-Of-Order Sample Disabled", func(t *testing.T) { s := teststorage.New(t) - t.Cleanup(func() { - _ = s.Close() - }) + t.Cleanup(func() { _ = s.Close() }) runScrapeLoopTest(t, s, false) }) @@ -118,19 +125,16 @@ func TestStorageHandlesOutOfOrderTimestamps(t *testing.T) { // Test with specific OutOfOrderTimeWindow (600000) t.Run("Out-Of-Order Sample Enabled", func(t *testing.T) { s := teststorage.New(t, 600000) - t.Cleanup(func() { - _ = s.Close() - }) + t.Cleanup(func() { _ = s.Close() }) runScrapeLoopTest(t, s, true) }) } func runScrapeLoopTest(t *testing.T, s *teststorage.TestStorage, expectOutOfOrder bool) { - // Create an appender for adding samples to the storage. - app := s.Appender(context.Background()) - capp := &collectResultAppender{next: app} - sl := newBasicScrapeLoop(t, context.Background(), nil, func(context.Context) storage.Appender { return capp }, 0) + sl, _, appTest := newTestScrapeLoop(t) + // Inject storage, so we can query later. + appTest.Next = s // Current time for generating timestamps. now := time.Now() @@ -141,37 +145,35 @@ func runScrapeLoopTest(t *testing.T, s *teststorage.TestStorage, expectOutOfOrde timestampOutOfOrder := now.Add(-5 * time.Minute) timestampInorder2 := now.Add(5 * time.Minute) - slApp := sl.appender(context.Background()) - _, _, _, err := sl.append(slApp, []byte(`metric_total{a="1",b="1"} 1`), "text/plain", timestampInorder1) + slApp := sl.appender() + _, _, _, err := slApp.append([]byte(`metric_total{a="1",b="1"} 1`), "text/plain", timestampInorder1) require.NoError(t, err) - _, _, _, err = sl.append(slApp, []byte(`metric_total{a="1",b="1"} 2`), "text/plain", timestampOutOfOrder) + _, _, _, err = slApp.append([]byte(`metric_total{a="1",b="1"} 2`), "text/plain", timestampOutOfOrder) require.NoError(t, err) - _, _, _, err = sl.append(slApp, []byte(`metric_total{a="1",b="1"} 3`), "text/plain", timestampInorder2) + _, _, _, err = slApp.append([]byte(`metric_total{a="1",b="1"} 3`), "text/plain", timestampInorder2) require.NoError(t, err) require.NoError(t, slApp.Commit()) // Query the samples back from the storage. - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() q, err := s.Querier(time.Time{}.UnixNano(), time.Now().UnixNano()) require.NoError(t, err) - defer q.Close() + t.Cleanup(func() { _ = q.Close() }) // Use a matcher to filter the metric name. - series := q.Select(ctx, false, nil, labels.MustNewMatcher(labels.MatchRegexp, "__name__", "metric_total")) + series := q.Select(t.Context(), false, nil, labels.MustNewMatcher(labels.MatchRegexp, "__name__", "metric_total")) - var results []floatSample + var results []sample for series.Next() { it := series.At().Iterator(nil) for it.Next() == chunkenc.ValFloat { t, v := it.At() - results = append(results, floatSample{ - metric: series.At().Labels(), - t: t, - f: v, + results = append(results, sample{ + L: series.At().Labels(), + T: t, + V: v, }) } require.NoError(t, it.Err()) @@ -179,16 +181,16 @@ func runScrapeLoopTest(t *testing.T, s *teststorage.TestStorage, expectOutOfOrde require.NoError(t, series.Err()) // Define the expected results - want := []floatSample{ + want := []sample{ { - metric: labels.FromStrings("__name__", "metric_total", "a", "1", "b", "1"), - t: timestamp.FromTime(timestampInorder1), - f: 1, + L: labels.FromStrings("__name__", "metric_total", "a", "1", "b", "1"), + T: timestamp.FromTime(timestampInorder1), + V: 1, }, { - metric: labels.FromStrings("__name__", "metric_total", "a", "1", "b", "1"), - t: timestamp.FromTime(timestampInorder2), - f: 3, + L: labels.FromStrings("__name__", "metric_total", "a", "1", "b", "1"), + T: timestamp.FromTime(timestampInorder2), + V: 3, }, } @@ -200,7 +202,7 @@ func runScrapeLoopTest(t *testing.T, s *teststorage.TestStorage, expectOutOfOrde } // Regression test against https://github.com/prometheus/prometheus/issues/15831. -func TestScrapeAppendMetadataUpdate(t *testing.T) { +func TestScrapeAppendMetadata(t *testing.T) { const ( scrape1 = `# TYPE test_metric counter # HELP test_metric some help text @@ -223,60 +225,68 @@ test_metric2{foo="bar"} 22 # EOF` ) - // Create an appender for adding samples to the storage. - capp := &collectResultAppender{next: nopAppender{}} - sl := newBasicScrapeLoop(t, context.Background(), nil, func(context.Context) storage.Appender { return capp }, 0) + sl, _, appTest := newTestScrapeLoop(t) now := time.Now() - slApp := sl.appender(context.Background()) - _, _, _, err := sl.append(slApp, []byte(scrape1), "application/openmetrics-text", now) + slApp := sl.appender() + _, _, _, err := slApp.append([]byte(scrape1), "application/openmetrics-text", now) require.NoError(t, err) require.NoError(t, slApp.Commit()) - testutil.RequireEqualWithOptions(t, []metadataEntry{ - {metric: labels.FromStrings("__name__", "test_metric_total"), m: metadata.Metadata{Type: "counter", Unit: "metric", Help: "some help text"}}, - {metric: labels.FromStrings("__name__", "test_metric2", "foo", "bar"), m: metadata.Metadata{Type: "gauge", Unit: "", Help: "other help text"}}, - }, capp.resultMetadata, []cmp.Option{cmp.Comparer(metadataEntryEqual)}) - capp.resultMetadata = nil + testutil.RequireEqual(t, []sample{ + {L: labels.FromStrings("__name__", "test_metric_total"), M: metadata.Metadata{Type: "counter", Unit: "metric", Help: "some help text"}}, + {L: labels.FromStrings("__name__", "test_metric2", "foo", "bar"), M: metadata.Metadata{Type: "gauge", Unit: "", Help: "other help text"}}, + {L: labels.FromStrings("__name__", "test_metric3_metric4", "foo", "bar")}, + }, appTest.ResultMetadata()) + appTest.ResultReset() - // Next (the same) scrape should not add new metadata entries. - slApp = sl.appender(context.Background()) - _, _, _, err = sl.append(slApp, []byte(scrape1), "application/openmetrics-text", now.Add(15*time.Second)) + // Next (the same) scrape MUST also add new metadata entries. + // NOTE: This is an important change since Appender v1. + slApp = sl.appender() + _, _, _, err = slApp.append([]byte(scrape1), "application/openmetrics-text", now.Add(15*time.Second)) require.NoError(t, err) require.NoError(t, slApp.Commit()) - testutil.RequireEqualWithOptions(t, []metadataEntry(nil), capp.resultMetadata, []cmp.Option{cmp.Comparer(metadataEntryEqual)}) + testutil.RequireEqual(t, []sample{ + {L: labels.FromStrings("__name__", "test_metric_total"), M: metadata.Metadata{Type: "counter", Unit: "metric", Help: "some help text"}}, + {L: labels.FromStrings("__name__", "test_metric2", "foo", "bar"), M: metadata.Metadata{Type: "gauge", Unit: "", Help: "other help text"}}, + {L: labels.FromStrings("__name__", "test_metric3_metric4", "foo", "bar")}, + }, appTest.ResultMetadata()) + appTest.ResultReset() - slApp = sl.appender(context.Background()) - _, _, _, err = sl.append(slApp, []byte(scrape2), "application/openmetrics-text", now.Add(15*time.Second)) + slApp = sl.appender() + _, _, _, err = slApp.append([]byte(scrape2), "application/openmetrics-text", now.Add(15*time.Second)) require.NoError(t, err) require.NoError(t, slApp.Commit()) - testutil.RequireEqualWithOptions(t, []metadataEntry{ - {metric: labels.FromStrings("__name__", "test_metric_total"), m: metadata.Metadata{Type: "counter", Unit: "metric", Help: "different help text"}}, // Here, technically we should have no unit, but it's a known limitation of the current implementation. - {metric: labels.FromStrings("__name__", "test_metric2", "foo", "bar"), m: metadata.Metadata{Type: "gauge", Unit: "metric2", Help: "other help text"}}, - }, capp.resultMetadata, []cmp.Option{cmp.Comparer(metadataEntryEqual)}) + testutil.RequireEqual(t, []sample{ + {L: labels.FromStrings("__name__", "test_metric_total"), M: metadata.Metadata{Type: "counter", Unit: "metric", Help: "different help text"}}, // Here, technically we should have no unit, but it's a known limitation of the current implementation. + {L: labels.FromStrings("__name__", "test_metric2", "foo", "bar"), M: metadata.Metadata{Type: "gauge", Unit: "metric2", Help: "other help text"}}, + {L: labels.FromStrings("__name__", "test_metric3_metric4", "foo", "bar")}, // Stale marker. + }, appTest.ResultMetadata()) + appTest.ResultReset() + + slApp = sl.appender() + _, _, _, err = slApp.append([]byte(scrape2), "application/openmetrics-text", now.Add(15*time.Second)) + require.NoError(t, err) + require.NoError(t, slApp.Commit()) + testutil.RequireEqual(t, []sample{ + {L: labels.FromStrings("__name__", "test_metric_total"), M: metadata.Metadata{Type: "counter", Unit: "metric", Help: "different help text"}}, // Here, technically we should have no unit, but it's a known limitation of the current implementation. + {L: labels.FromStrings("__name__", "test_metric2", "foo", "bar"), M: metadata.Metadata{Type: "gauge", Unit: "metric2", Help: "other help text"}}, + }, appTest.ResultMetadata()) } -type nopScraper struct { - scraper -} +func TestScrapeReportMetadata(t *testing.T) { + sl, _, appTest := newTestScrapeLoop(t) + slApp := sl.appender() -func (nopScraper) Report(time.Time, time.Duration, error) {} - -func TestScrapeReportMetadataUpdate(t *testing.T) { - // Create an appender for adding samples to the storage. - capp := &collectResultAppender{next: nopAppender{}} - sl := newBasicScrapeLoop(t, context.Background(), nopScraper{}, func(context.Context) storage.Appender { return capp }, 0) now := time.Now() - slApp := sl.appender(context.Background()) - require.NoError(t, sl.report(slApp, now, 2*time.Second, 1, 1, 1, 512, nil)) require.NoError(t, slApp.Commit()) - testutil.RequireEqualWithOptions(t, []metadataEntry{ - {metric: labels.FromStrings("__name__", "up"), m: scrapeHealthMetric.Metadata}, - {metric: labels.FromStrings("__name__", "scrape_duration_seconds"), m: scrapeDurationMetric.Metadata}, - {metric: labels.FromStrings("__name__", "scrape_samples_scraped"), m: scrapeSamplesMetric.Metadata}, - {metric: labels.FromStrings("__name__", "scrape_samples_post_metric_relabeling"), m: samplesPostRelabelMetric.Metadata}, - {metric: labels.FromStrings("__name__", "scrape_series_added"), m: scrapeSeriesAddedMetric.Metadata}, - }, capp.resultMetadata, []cmp.Option{cmp.Comparer(metadataEntryEqual)}) + testutil.RequireEqual(t, []sample{ + {L: labels.FromStrings("__name__", "up"), M: scrapeHealthMetric.Metadata}, + {L: labels.FromStrings("__name__", "scrape_duration_seconds"), M: scrapeDurationMetric.Metadata}, + {L: labels.FromStrings("__name__", "scrape_samples_scraped"), M: scrapeSamplesMetric.Metadata}, + {L: labels.FromStrings("__name__", "scrape_samples_post_metric_relabeling"), M: samplesPostRelabelMetric.Metadata}, + {L: labels.FromStrings("__name__", "scrape_series_added"), M: scrapeSeriesAddedMetric.Metadata}, + }, appTest.ResultMetadata()) } func TestIsSeriesPartOfFamily(t *testing.T) { @@ -352,7 +362,7 @@ func TestDroppedTargetsList(t *testing.T) { }, }, } - sp, _ = newScrapePool(cfg, app, 0, nil, nil, &Options{}, newTestScrapeMetrics(t)) + sp, _ = newScrapePool(cfg, nil, app, 0, nil, nil, &Options{}, newTestScrapeMetrics(t)) expectedLabelSetString = "{__address__=\"127.0.0.1:9090\", __scrape_interval__=\"0s\", __scrape_timeout__=\"0s\", job=\"dropMe\"}" expectedLength = 2 ) @@ -536,7 +546,7 @@ func TestScrapePoolReload(t *testing.T) { reg, metrics := newTestRegistryAndScrapeMetrics(t) sp := &scrapePool{ - appendable: &nopAppendable{}, + appendableV2: &nopAppendable{}, activeTargets: map[uint64]*Target{}, loops: map[uint64]loop{}, newLoop: newLoop, @@ -551,9 +561,8 @@ func TestScrapePoolReload(t *testing.T) { // one terminated. for i := range numTargets { - labels := labels.FromStrings(model.AddressLabel, fmt.Sprintf("example.com:%d", i)) t := &Target{ - labels: labels, + labels: labels.FromStrings(model.AddressLabel, fmt.Sprintf("example.com:%d", i)), scrapeConfig: &config.ScrapeConfig{}, } l := &testLoop{} @@ -577,7 +586,7 @@ func TestScrapePoolReload(t *testing.T) { reloadTime := time.Now() go func() { - sp.reload(reloadCfg) + _ = sp.reload(reloadCfg) close(done) }() @@ -620,7 +629,7 @@ func TestScrapePoolReloadPreserveRelabeledIntervalTimeout(t *testing.T) { } reg, metrics := newTestRegistryAndScrapeMetrics(t) sp := &scrapePool{ - appendable: &nopAppendable{}, + appendableV2: &nopAppendable{}, activeTargets: map[uint64]*Target{ 1: { labels: labels.FromStrings(model.ScrapeIntervalLabel, "5s", model.ScrapeTimeoutLabel, "3s"), @@ -681,7 +690,7 @@ func TestScrapePoolTargetLimit(t *testing.T) { return l } sp := &scrapePool{ - appendable: &nopAppendable{}, + appendableV2: &nopAppendable{}, activeTargets: map[uint64]*Target{}, loops: map[uint64]loop{}, newLoop: newLoop, @@ -802,7 +811,7 @@ func TestScrapePoolAppender(t *testing.T) { MetricNameEscapingScheme: model.AllowUTF8, } app := &nopAppendable{} - sp, _ := newScrapePool(cfg, app, 0, nil, nil, &Options{}, newTestScrapeMetrics(t)) + sp, _ := newScrapePool(cfg, nil, app, 0, nil, nil, &Options{}, newTestScrapeMetrics(t)) loop := sp.newLoop(scrapeLoopOptions{ target: &Target{}, @@ -810,13 +819,13 @@ func TestScrapePoolAppender(t *testing.T) { appl, ok := loop.(*scrapeLoop) require.True(t, ok, "Expected scrapeLoop but got %T", loop) - wrapped := appender(appl.appender(context.Background()), 0, 0, histogram.ExponentialSchemaMax) + wrapped := appender(appl.appendableV2.AppenderV2(context.Background()), 0, 0, histogram.ExponentialSchemaMax) tl, ok := wrapped.(*timeLimitAppender) require.True(t, ok, "Expected timeLimitAppender but got %T", wrapped) - _, ok = tl.Appender.(nopAppender) - require.True(t, ok, "Expected base appender but got %T", tl.Appender) + _, ok = tl.AppenderV2.(nopAppender) + require.True(t, ok, "Expected base appender but got %T", tl.AppenderV2) sampleLimit := 100 loop = sp.newLoop(scrapeLoopOptions{ @@ -826,47 +835,47 @@ func TestScrapePoolAppender(t *testing.T) { appl, ok = loop.(*scrapeLoop) require.True(t, ok, "Expected scrapeLoop but got %T", loop) - wrapped = appender(appl.appender(context.Background()), sampleLimit, 0, histogram.ExponentialSchemaMax) + wrapped = appender(appl.appendableV2.AppenderV2(context.Background()), sampleLimit, 0, histogram.ExponentialSchemaMax) sl, ok := wrapped.(*limitAppender) require.True(t, ok, "Expected limitAppender but got %T", wrapped) - tl, ok = sl.Appender.(*timeLimitAppender) - require.True(t, ok, "Expected timeLimitAppender but got %T", sl.Appender) + tl, ok = sl.AppenderV2.(*timeLimitAppender) + require.True(t, ok, "Expected timeLimitAppender but got %T", sl.AppenderV2) - _, ok = tl.Appender.(nopAppender) - require.True(t, ok, "Expected base appender but got %T", tl.Appender) + _, ok = tl.AppenderV2.(nopAppender) + require.True(t, ok, "Expected base appender but got %T", tl.AppenderV2) - wrapped = appender(appl.appender(context.Background()), sampleLimit, 100, histogram.ExponentialSchemaMax) + wrapped = appender(appl.appendableV2.AppenderV2(context.Background()), sampleLimit, 100, histogram.ExponentialSchemaMax) bl, ok := wrapped.(*bucketLimitAppender) require.True(t, ok, "Expected bucketLimitAppender but got %T", wrapped) - sl, ok = bl.Appender.(*limitAppender) + sl, ok = bl.AppenderV2.(*limitAppender) require.True(t, ok, "Expected limitAppender but got %T", bl) - tl, ok = sl.Appender.(*timeLimitAppender) - require.True(t, ok, "Expected timeLimitAppender but got %T", sl.Appender) + tl, ok = sl.AppenderV2.(*timeLimitAppender) + require.True(t, ok, "Expected timeLimitAppender but got %T", sl.AppenderV2) - _, ok = tl.Appender.(nopAppender) - require.True(t, ok, "Expected base appender but got %T", tl.Appender) + _, ok = tl.AppenderV2.(nopAppender) + require.True(t, ok, "Expected base appender but got %T", tl.AppenderV2) - wrapped = appender(appl.appender(context.Background()), sampleLimit, 100, 0) + wrapped = appender(appl.appendableV2.AppenderV2(context.Background()), sampleLimit, 100, 0) ml, ok := wrapped.(*maxSchemaAppender) require.True(t, ok, "Expected maxSchemaAppender but got %T", wrapped) - bl, ok = ml.Appender.(*bucketLimitAppender) + bl, ok = ml.AppenderV2.(*bucketLimitAppender) require.True(t, ok, "Expected bucketLimitAppender but got %T", wrapped) - sl, ok = bl.Appender.(*limitAppender) + sl, ok = bl.AppenderV2.(*limitAppender) require.True(t, ok, "Expected limitAppender but got %T", bl) - tl, ok = sl.Appender.(*timeLimitAppender) - require.True(t, ok, "Expected timeLimitAppender but got %T", sl.Appender) + tl, ok = sl.AppenderV2.(*timeLimitAppender) + require.True(t, ok, "Expected timeLimitAppender but got %T", sl.AppenderV2) - _, ok = tl.Appender.(nopAppender) - require.True(t, ok, "Expected base appender but got %T", tl.Appender) + _, ok = tl.AppenderV2.(nopAppender) + require.True(t, ok, "Expected base appender but got %T", tl.AppenderV2) } func TestScrapePoolRaces(t *testing.T) { @@ -881,7 +890,7 @@ func TestScrapePoolRaces(t *testing.T) { MetricNameEscapingScheme: model.AllowUTF8, } } - sp, _ := newScrapePool(newConfig(), &nopAppendable{}, 0, nil, nil, &Options{}, newTestScrapeMetrics(t)) + sp, _ := newScrapePool(newConfig(), nil, &nopAppendable{}, 0, nil, nil, &Options{}, newTestScrapeMetrics(t)) tgts := []*targetgroup.Group{ { Targets: []model.LabelSet{ @@ -907,7 +916,7 @@ func TestScrapePoolRaces(t *testing.T) { for range 20 { time.Sleep(10 * time.Millisecond) - sp.reload(newConfig()) + _ = sp.reload(newConfig()) } sp.stop() } @@ -925,7 +934,7 @@ func TestScrapePoolScrapeLoopsStarted(t *testing.T) { return l } sp := &scrapePool{ - appendable: &nopAppendable{}, + appendableV2: &nopAppendable{}, activeTargets: map[uint64]*Target{}, loops: map[uint64]loop{}, newLoop: newLoop, @@ -964,48 +973,57 @@ func TestScrapePoolScrapeLoopsStarted(t *testing.T) { } } -func newBasicScrapeLoop(t testing.TB, ctx context.Context, scraper scraper, app func(ctx context.Context) storage.Appender, interval time.Duration) *scrapeLoop { - return newBasicScrapeLoopWithFallback(t, ctx, scraper, app, interval, "") -} +// newTestScrapeLoop is the initial scrape loop for all tests. +// It returns scrapeLoop but also mock scraper (for injections) and appender +// (for injections and append result testing). +// +// NOTE: Try to NOT add more parameter to this function. Try to NOT add more +// newTestScrapeLoop-like constructors. It should be flexible enough with scrapeLoop +// used for initial options. +func newTestScrapeLoop(t testing.TB, opts ...func(sl *scrapeLoop)) (_ *scrapeLoop, scraper *testScraper, appTest *teststorage.Appender) { + scraper = &testScraper{} + appTest = teststorage.NewAppender() -func newBasicScrapeLoopWithFallback(t testing.TB, ctx context.Context, scraper scraper, app func(ctx context.Context) storage.Appender, interval time.Duration, fallback string) *scrapeLoop { - return newScrapeLoop(ctx, - scraper, - nil, nil, - nopMutator, - nopMutator, - app, - nil, - labels.NewSymbolTable(), - 0, - true, - false, - true, - 0, 0, histogram.ExponentialSchemaMax, - nil, - interval, - time.Hour, - false, - false, - false, - false, - false, - false, - true, - nil, - false, - newTestScrapeMetrics(t), - false, - model.UTF8Validation, - model.NoEscaping, - fallback, - ) + ctx := t.Context() + sl := &scrapeLoop{ + appendableV2: appTest, + sampleMutator: nopMutator, + reportSampleMutator: nopMutator, + validationScheme: model.UTF8Validation, + symbolTable: labels.NewSymbolTable(), + metrics: newTestScrapeMetrics(t), + scrapeLoopOptions: scrapeLoopOptions{ + interval: 10 * time.Millisecond, + bucketLimit: int(histogram.ExponentialSchemaMax), + timeout: 1 * time.Hour, + honorTimestamps: true, + enableCompression: true, + }, + appendMetadataToWAL: true, // Tests assumes it's enabled. + } + for _, o := range opts { + o(sl) + } + // Use sl.ctx as context injection. + if sl.ctx != nil { + ctx = sl.ctx + } + + // Validate user opts for convenience. + require.Nil(t, sl.parentCtx, "newTestScrapeLoop does not support injecting non-ctx contexts") + require.Nil(t, sl.appenderCtx, "newTestScrapeLoop does not support injecting non-ctx contexts") + require.Nil(t, sl.cancel, "newTestScrapeLoop does not support injecting non-ctx contexts") + require.Nil(t, sl.appendableV1, "newTestScrapeLoop does not support appender v1 flow") + require.Nil(t, sl.scraper, "newTestScrapeLoop does not support injecting scraper, it's mocked, use returned scraper") + sl.scraper = scraper + sl.init(ctx, true) + return sl, scraper, appTest } func TestScrapeLoopStopBeforeRun(t *testing.T) { t.Parallel() - scraper := &testScraper{} - sl := newBasicScrapeLoop(t, context.Background(), scraper, nil, 1) + + sl, scraper, _ := newTestScrapeLoop(t) // The scrape pool synchronizes on stopping scrape loops. However, new scrape // loops are started asynchronously. Thus it's possible, that a loop is stopped @@ -1053,26 +1071,22 @@ func TestScrapeLoopStopBeforeRun(t *testing.T) { func nopMutator(l labels.Labels) labels.Labels { return l } func TestScrapeLoopStop(t *testing.T) { - var ( - signal = make(chan struct{}, 1) - appender = &collectResultAppender{} - scraper = &testScraper{} - app = func(context.Context) storage.Appender { return appender } - ) + signal := make(chan struct{}, 1) - // Since we're writing samples directly below we need to provide a protocol fallback. - sl := newBasicScrapeLoopWithFallback(t, context.Background(), scraper, app, 10*time.Millisecond, "text/plain") + sl, scraper, appTest := newTestScrapeLoop(t, func(sl *scrapeLoop) { + // Since we're writing samples directly below we need to provide a protocol fallback. + sl.fallbackScrapeProtocol = "text/plain" + }) // Terminate loop after 2 scrapes. numScrapes := 0 - scraper.scrapeFunc = func(ctx context.Context, w io.Writer) error { numScrapes++ if numScrapes == 2 { go sl.stop() <-sl.ctx.Done() } - w.Write([]byte("metric_a 42\n")) + _, _ = w.Write([]byte("metric_a 42\n")) return ctx.Err() } @@ -1089,21 +1103,21 @@ func TestScrapeLoopStop(t *testing.T) { // We expected 1 actual sample for each scrape plus 5 for report samples. // At least 2 scrapes were made, plus the final stale markers. - require.GreaterOrEqual(t, len(appender.resultFloats), 6*3, "Expected at least 3 scrapes with 6 samples each.") - require.Zero(t, len(appender.resultFloats)%6, "There is a scrape with missing samples.") + require.GreaterOrEqual(t, len(appTest.ResultSamples), 6*3, "Expected at least 3 scrapes with 6 samples each.") + require.Zero(t, len(appTest.ResultSamples)%6, "There is a scrape with missing samples.") // All samples in a scrape must have the same timestamp. var ts int64 - for i, s := range appender.resultFloats { + for i, s := range appTest.ResultSamples { switch { case i%6 == 0: - ts = s.t - case s.t != ts: + ts = s.T + case s.T != ts: t.Fatalf("Unexpected multiple timestamps within single scrape") } } // All samples from the last scrape must be stale markers. - for _, s := range appender.resultFloats[len(appender.resultFloats)-5:] { - require.True(t, value.IsStaleNaN(s.f), "Appended last sample not as expected. Wanted: stale NaN Got: %x", math.Float64bits(s.f)) + for _, s := range appTest.ResultSamples[len(appTest.ResultSamples)-5:] { + require.True(t, value.IsStaleNaN(s.V), "Appended last sample not as expected. Wanted: stale NaN Got: %x", math.Float64bits(s.V)) } } @@ -1112,45 +1126,12 @@ func TestScrapeLoopRun(t *testing.T) { var ( signal = make(chan struct{}, 1) errc = make(chan error) - - scraper = &testScraper{} - app = func(context.Context) storage.Appender { return &nopAppender{} } - scrapeMetrics = newTestScrapeMetrics(t) - ) - - ctx, cancel := context.WithCancel(context.Background()) - sl := newScrapeLoop(ctx, - scraper, - nil, nil, - nopMutator, - nopMutator, - app, - nil, - nil, - 0, - true, - false, - true, - 0, 0, histogram.ExponentialSchemaMax, - nil, - time.Second, - time.Hour, - false, - false, - false, - false, - false, - false, - false, - nil, - false, - scrapeMetrics, - false, - model.UTF8Validation, - model.NoEscaping, - "", ) + ctx, cancel := context.WithCancel(t.Context()) + sl, scraper, _ := newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.ctx = ctx + }) // The loop must terminate during the initial offset if the context // is canceled. scraper.offsetDur = time.Hour @@ -1186,9 +1167,11 @@ func TestScrapeLoopRun(t *testing.T) { return nil } - ctx, cancel = context.WithCancel(context.Background()) - sl = newBasicScrapeLoop(t, ctx, scraper, app, time.Second) - sl.timeout = 100 * time.Millisecond + ctx, cancel = context.WithCancel(t.Context()) + sl, scraper, _ = newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.ctx = ctx + sl.timeout = 100 * time.Millisecond + }) go func() { sl.run(errc) @@ -1222,16 +1205,14 @@ func TestScrapeLoopForcedErr(t *testing.T) { var ( signal = make(chan struct{}, 1) errc = make(chan error) - - scraper = &testScraper{} - app = func(context.Context) storage.Appender { return &nopAppender{} } ) - ctx, cancel := context.WithCancel(context.Background()) - sl := newBasicScrapeLoop(t, ctx, scraper, app, time.Second) - + ctx, cancel := context.WithCancel(t.Context()) forcedErr := errors.New("forced err") - sl.setForcedError(forcedErr) + sl, scraper, _ := newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.ctx = ctx + sl.setForcedError(forcedErr) + }) scraper.scrapeFunc = func(context.Context, io.Writer) error { require.FailNow(t, "Should not be scraped.") @@ -1259,50 +1240,10 @@ func TestScrapeLoopForcedErr(t *testing.T) { } func TestScrapeLoopMetadata(t *testing.T) { - var ( - signal = make(chan struct{}) - scraper = &testScraper{} - scrapeMetrics = newTestScrapeMetrics(t) - cache = newScrapeCache(scrapeMetrics) - ) - defer close(signal) + sl, _, _ := newTestScrapeLoop(t) - ctx, cancel := context.WithCancel(context.Background()) - sl := newScrapeLoop(ctx, - scraper, - nil, nil, - nopMutator, - nopMutator, - func(context.Context) storage.Appender { return nopAppender{} }, - cache, - labels.NewSymbolTable(), - 0, - true, - false, - true, - 0, 0, histogram.ExponentialSchemaMax, - nil, - 0, - 0, - false, - false, - false, - false, - false, - false, - false, - nil, - false, - scrapeMetrics, - false, - model.UTF8Validation, - model.NoEscaping, - "", - ) - defer cancel() - - slApp := sl.appender(ctx) - total, _, _, err := sl.append(slApp, []byte(`# TYPE test_metric counter + slApp := sl.appender() + total, _, _, err := slApp.append([]byte(`# TYPE test_metric counter # HELP test_metric some help text # UNIT test_metric metric test_metric_total 1 @@ -1313,50 +1254,38 @@ test_metric_total 1 require.NoError(t, slApp.Commit()) require.Equal(t, 1, total) - md, ok := cache.GetMetadata("test_metric") + md, ok := sl.cache.GetMetadata("test_metric") require.True(t, ok, "expected metadata to be present") require.Equal(t, model.MetricTypeCounter, md.Type, "unexpected metric type") require.Equal(t, "some help text", md.Help) require.Equal(t, "metric", md.Unit) - md, ok = cache.GetMetadata("test_metric_no_help") + md, ok = sl.cache.GetMetadata("test_metric_no_help") require.True(t, ok, "expected metadata to be present") require.Equal(t, model.MetricTypeGauge, md.Type, "unexpected metric type") require.Empty(t, md.Help) require.Empty(t, md.Unit) - md, ok = cache.GetMetadata("test_metric_no_type") + md, ok = sl.cache.GetMetadata("test_metric_no_type") require.True(t, ok, "expected metadata to be present") require.Equal(t, model.MetricTypeUnknown, md.Type, "unexpected metric type") require.Equal(t, "other help text", md.Help) require.Empty(t, md.Unit) } -func simpleTestScrapeLoop(t testing.TB) (context.Context, *scrapeLoop) { - // Need a full storage for correct Add/AddFast semantics. - s := teststorage.New(t) - t.Cleanup(func() { s.Close() }) - - ctx, cancel := context.WithCancel(context.Background()) - sl := newBasicScrapeLoop(t, ctx, &testScraper{}, s.Appender, 0) - t.Cleanup(func() { cancel() }) - - return ctx, sl -} - func TestScrapeLoopSeriesAdded(t *testing.T) { - ctx, sl := simpleTestScrapeLoop(t) + sl, _, _ := newTestScrapeLoop(t) - slApp := sl.appender(ctx) - total, added, seriesAdded, err := sl.append(slApp, []byte("test_metric 1\n"), "text/plain", time.Time{}) + slApp := sl.appender() + total, added, seriesAdded, err := slApp.append([]byte("test_metric 1\n"), "text/plain", time.Time{}) require.NoError(t, err) require.NoError(t, slApp.Commit()) require.Equal(t, 1, total) require.Equal(t, 1, added) require.Equal(t, 1, seriesAdded) - slApp = sl.appender(ctx) - total, added, seriesAdded, err = sl.append(slApp, []byte("test_metric 1\n"), "text/plain", time.Time{}) + slApp = sl.appender() + total, added, seriesAdded, err = slApp.append([]byte("test_metric 1\n"), "text/plain", time.Time{}) require.NoError(t, slApp.Commit()) require.NoError(t, err) require.Equal(t, 1, total) @@ -1365,10 +1294,6 @@ func TestScrapeLoopSeriesAdded(t *testing.T) { } func TestScrapeLoopFailWithInvalidLabelsAfterRelabel(t *testing.T) { - s := teststorage.New(t) - defer s.Close() - ctx := t.Context() - target := &Target{ labels: labels.FromStrings("pod_label_invalid_012\xff", "test"), } @@ -1379,13 +1304,14 @@ func TestScrapeLoopFailWithInvalidLabelsAfterRelabel(t *testing.T) { Replacement: "$1", NameValidationScheme: model.UTF8Validation, }} - sl := newBasicScrapeLoop(t, ctx, &testScraper{}, s.Appender, 0) - sl.sampleMutator = func(l labels.Labels) labels.Labels { - return mutateSampleLabels(l, target, true, relabelConfig) - } + sl, _, _ := newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.sampleMutator = func(l labels.Labels) labels.Labels { + return mutateSampleLabels(l, target, true, relabelConfig) + } + }) - slApp := sl.appender(ctx) - total, added, seriesAdded, err := sl.append(slApp, []byte("test_metric 1\n"), "text/plain", time.Time{}) + slApp := sl.appender() + total, added, seriesAdded, err := slApp.append([]byte("test_metric 1\n"), "text/plain", time.Time{}) require.ErrorContains(t, err, "invalid metric name or label names") require.NoError(t, slApp.Rollback()) require.Equal(t, 1, total) @@ -1394,17 +1320,12 @@ func TestScrapeLoopFailWithInvalidLabelsAfterRelabel(t *testing.T) { } func TestScrapeLoopFailLegacyUnderUTF8(t *testing.T) { - // Test that scrapes fail when default validation is utf8 but scrape config is - // legacy. - s := teststorage.New(t) - defer s.Close() - ctx := t.Context() + sl, _, _ := newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.validationScheme = model.LegacyValidation + }) - sl := newBasicScrapeLoop(t, ctx, &testScraper{}, s.Appender, 0) - sl.validationScheme = model.LegacyValidation - - slApp := sl.appender(ctx) - total, added, seriesAdded, err := sl.append(slApp, []byte("{\"test.metric\"} 1\n"), "text/plain", time.Time{}) + slApp := sl.appender() + total, added, seriesAdded, err := slApp.append([]byte("{\"test.metric\"} 1\n"), "text/plain", time.Time{}) require.ErrorContains(t, err, "invalid metric name or label names") require.NoError(t, slApp.Rollback()) require.Equal(t, 1, total) @@ -1412,10 +1333,12 @@ func TestScrapeLoopFailLegacyUnderUTF8(t *testing.T) { require.Equal(t, 0, seriesAdded) // When scrapeloop has validation set to UTF-8, the metric is allowed. - sl.validationScheme = model.UTF8Validation + sl, _, _ = newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.validationScheme = model.UTF8Validation + }) - slApp = sl.appender(ctx) - total, added, seriesAdded, err = sl.append(slApp, []byte("{\"test.metric\"} 1\n"), "text/plain", time.Time{}) + slApp = sl.appender() + total, added, seriesAdded, err = slApp.append([]byte("{\"test.metric\"} 1\n"), "text/plain", time.Time{}) require.NoError(t, err) require.Equal(t, 1, total) require.Equal(t, 1, added) @@ -1434,12 +1357,12 @@ func readTextParseTestMetrics(t testing.TB) []byte { func makeTestGauges(n int) []byte { sb := bytes.Buffer{} - fmt.Fprintf(&sb, "# TYPE metric_a gauge\n") - fmt.Fprintf(&sb, "# HELP metric_a help text\n") + _, _ = fmt.Fprintf(&sb, "# TYPE metric_a gauge\n") + _, _ = fmt.Fprintf(&sb, "# HELP metric_a help text\n") for i := range n { - fmt.Fprintf(&sb, "metric_a{foo=\"%d\",bar=\"%d\"} 1\n", i, i*100) + _, _ = fmt.Fprintf(&sb, "metric_a{foo=\"%d\",bar=\"%d\"} 1\n", i, i*100) } - fmt.Fprintf(&sb, "# EOF\n") + _, _ = fmt.Fprintf(&sb, "# EOF\n") return sb.Bytes() } @@ -1536,16 +1459,21 @@ func BenchmarkScrapeLoopAppend(b *testing.B) { {name: "PromProto", contentType: "application/vnd.google.protobuf", parsable: metricsProto}, } { b.Run(fmt.Sprintf("fmt=%v", bcase.name), func(b *testing.B) { - ctx, sl := simpleTestScrapeLoop(b) + // Need a full storage for correct Add/AddFast semantics. + s := teststorage.New(b) + b.Cleanup(func() { _ = s.Close() }) - slApp := sl.appender(ctx) + sl, _, _ := newTestScrapeLoop(b, func(sl *scrapeLoop) { + sl.appendableV2 = s + }) + slApp := sl.appender() ts := time.Time{} b.ReportAllocs() b.ResetTimer() for b.Loop() { ts = ts.Add(time.Second) - _, _, _, err := sl.append(slApp, bcase.parsable, bcase.contentType, ts) + _, _, _, err := slApp.append(bcase.parsable, bcase.contentType, ts) if err != nil { b.Fatal(err) } @@ -1558,28 +1486,26 @@ func BenchmarkScrapeLoopAppend(b *testing.B) { func TestSetOptionsHandlingStaleness(t *testing.T) { s := teststorage.New(t, 600000) - defer s.Close() + t.Cleanup(func() { _ = s.Close() }) signal := make(chan struct{}, 1) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(t.Context()) defer cancel() // Function to run the scrape loop runScrapeLoop := func(ctx context.Context, t *testing.T, cue int, action func(*scrapeLoop)) { - var ( - scraper = &testScraper{} - app = func(ctx context.Context) storage.Appender { - return s.Appender(ctx) - } - ) - sl := newBasicScrapeLoop(t, ctx, scraper, app, 10*time.Millisecond) + sl, scraper, _ := newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.ctx = ctx + sl.appendableV2 = s + }) + numScrapes := 0 scraper.scrapeFunc = func(_ context.Context, w io.Writer) error { numScrapes++ if numScrapes == cue { action(sl) } - fmt.Fprintf(w, "metric_a{a=\"1\",b=\"1\"} %d\n", 42+numScrapes) + _, _ = fmt.Fprintf(w, "metric_a{a=\"1\",b=\"1\"} %d\n", 42+numScrapes) return nil } sl.run(nil) @@ -1604,25 +1530,25 @@ func TestSetOptionsHandlingStaleness(t *testing.T) { t.Fatalf("Scrape wasn't stopped.") } - ctx1, cancel := context.WithCancel(context.Background()) + ctx1, cancel := context.WithCancel(t.Context()) defer cancel() q, err := s.Querier(0, time.Now().UnixNano()) require.NoError(t, err) - defer q.Close() + t.Cleanup(func() { _ = q.Close() }) series := q.Select(ctx1, false, nil, labels.MustNewMatcher(labels.MatchRegexp, "__name__", "metric_a")) - var results []floatSample + var results []sample for series.Next() { it := series.At().Iterator(nil) for it.Next() == chunkenc.ValFloat { t, v := it.At() - results = append(results, floatSample{ - metric: series.At().Labels(), - t: t, - f: v, + results = append(results, sample{ + L: series.At().Labels(), + T: t, + V: v, }) } require.NoError(t, it.Err()) @@ -1630,7 +1556,7 @@ func TestSetOptionsHandlingStaleness(t *testing.T) { require.NoError(t, series.Err()) var c int for _, s := range results { - if value.IsStaleNaN(s.f) { + if value.IsStaleNaN(s.V) { c++ } } @@ -1638,25 +1564,23 @@ func TestSetOptionsHandlingStaleness(t *testing.T) { } func TestScrapeLoopRunCreatesStaleMarkersOnFailedScrape(t *testing.T) { - appender := &collectResultAppender{} - var ( - signal = make(chan struct{}, 1) - scraper = &testScraper{} - app = func(context.Context) storage.Appender { return appender } - ) + signal := make(chan struct{}, 1) + + ctx, cancel := context.WithCancel(t.Context()) + sl, scraper, appTest := newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.ctx = ctx + // Since we're writing samples directly below we need to provide a protocol fallback. + sl.fallbackScrapeProtocol = "text/plain" + }) - ctx, cancel := context.WithCancel(context.Background()) - // Since we're writing samples directly below we need to provide a protocol fallback. - sl := newBasicScrapeLoopWithFallback(t, ctx, scraper, app, 10*time.Millisecond, "text/plain") // Succeed once, several failures, then stop. numScrapes := 0 - scraper.scrapeFunc = func(_ context.Context, w io.Writer) error { numScrapes++ switch numScrapes { case 1: - w.Write([]byte("metric_a 42\n")) + _, _ = w.Write([]byte("metric_a 42\n")) return nil case 5: cancel() @@ -1675,36 +1599,36 @@ func TestScrapeLoopRunCreatesStaleMarkersOnFailedScrape(t *testing.T) { require.FailNow(t, "Scrape wasn't stopped.") } - // 1 successfully scraped sample, 1 stale marker after first fail, 5 report samples for - // each scrape successful or not. - require.Len(t, appender.resultFloats, 27, "Appended samples not as expected:\n%s", appender) - require.Equal(t, 42.0, appender.resultFloats[0].f, "Appended first sample not as expected") - require.True(t, value.IsStaleNaN(appender.resultFloats[6].f), - "Appended second sample not as expected. Wanted: stale NaN Got: %x", math.Float64bits(appender.resultFloats[6].f)) + // 1 successfully scraped sample + // 1 stale marker after first fail + // 5x 5 report samples for each scrape successful or not. + require.Len(t, appTest.ResultSamples, 27, "Appended samples not as expected:\n%s", appTest) + require.Equal(t, 42.0, appTest.ResultSamples[0].V, "Appended first sample not as expected") + require.True(t, value.IsStaleNaN(appTest.ResultSamples[6].V), + "Appended second sample not as expected. Wanted: stale NaN Got: %x", math.Float64bits(appTest.ResultSamples[6].V)) } func TestScrapeLoopRunCreatesStaleMarkersOnParseFailure(t *testing.T) { - appender := &collectResultAppender{} - var ( - signal = make(chan struct{}, 1) - scraper = &testScraper{} - app = func(context.Context) storage.Appender { return appender } - numScrapes = 0 - ) + signal := make(chan struct{}, 1) - ctx, cancel := context.WithCancel(context.Background()) - // Since we're writing samples directly below we need to provide a protocol fallback. - sl := newBasicScrapeLoopWithFallback(t, ctx, scraper, app, 10*time.Millisecond, "text/plain") + ctx, cancel := context.WithCancel(t.Context()) + sl, scraper, appTest := newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.ctx = ctx + // Since we're writing samples directly below we need to provide a protocol fallback. + sl.fallbackScrapeProtocol = "text/plain" + }) // Succeed once, several failures, then stop. + numScrapes := 0 scraper.scrapeFunc = func(_ context.Context, w io.Writer) error { numScrapes++ + switch numScrapes { case 1: - w.Write([]byte("metric_a 42\n")) + _, _ = w.Write([]byte("metric_a 42\n")) return nil case 2: - w.Write([]byte("7&-\n")) + _, _ = w.Write([]byte("7&-\n")) return nil case 3: cancel() @@ -1719,46 +1643,46 @@ func TestScrapeLoopRunCreatesStaleMarkersOnParseFailure(t *testing.T) { select { case <-signal: + // TODO(bwplotka): Prone to flakiness, depend on atomic numScrapes. case <-time.After(5 * time.Second): require.FailNow(t, "Scrape wasn't stopped.") } - // 1 successfully scraped sample, 1 stale marker after first fail, 5 report samples for - // each scrape successful or not. - require.Len(t, appender.resultFloats, 17, "Appended samples not as expected:\n%s", appender) - require.Equal(t, 42.0, appender.resultFloats[0].f, "Appended first sample not as expected") - require.True(t, value.IsStaleNaN(appender.resultFloats[6].f), - "Appended second sample not as expected. Wanted: stale NaN Got: %x", math.Float64bits(appender.resultFloats[6].f)) + // 1 successfully scraped sample + // 1 stale marker after first fail + // 3x 5 report samples for each scrape successful or not. + require.Len(t, appTest.ResultSamples, 17, "Appended samples not as expected:\n%s", appTest) + require.Equal(t, 42.0, appTest.ResultSamples[0].V, "Appended first sample not as expected") + require.True(t, value.IsStaleNaN(appTest.ResultSamples[6].V), + "Appended second sample not as expected. Wanted: stale NaN Got: %x", math.Float64bits(appTest.ResultSamples[6].V)) } // If we have a target with sample_limit set and scrape initially works but then we hit the sample_limit error, // then we don't expect to see any StaleNaNs appended for the series that disappeared due to sample_limit error. func TestScrapeLoopRunCreatesStaleMarkersOnSampleLimit(t *testing.T) { - appender := &collectResultAppender{} - var ( - signal = make(chan struct{}, 1) - scraper = &testScraper{} - app = func(_ context.Context) storage.Appender { return appender } - numScrapes = 0 - ) + signal := make(chan struct{}, 1) - ctx, cancel := context.WithCancel(context.Background()) - // Since we're writing samples directly below we need to provide a protocol fallback. - sl := newBasicScrapeLoopWithFallback(t, ctx, scraper, app, 10*time.Millisecond, "text/plain") - sl.sampleLimit = 4 + ctx, cancel := context.WithCancel(t.Context()) + sl, scraper, appTest := newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.ctx = ctx + // Since we're writing samples directly below we need to provide a protocol fallback. + sl.fallbackScrapeProtocol = "text/plain" + sl.sampleLimit = 4 + }) // Succeed once, several failures, then stop. + numScrapes := 0 scraper.scrapeFunc = func(_ context.Context, w io.Writer) error { numScrapes++ switch numScrapes { case 1: - w.Write([]byte("metric_a 10\nmetric_b 10\nmetric_c 10\nmetric_d 10\n")) + _, _ = w.Write([]byte("metric_a 10\nmetric_b 10\nmetric_c 10\nmetric_d 10\n")) return nil case 2: - w.Write([]byte("metric_a 20\nmetric_b 20\nmetric_c 20\nmetric_d 20\nmetric_e 999\n")) + _, _ = w.Write([]byte("metric_a 20\nmetric_b 20\nmetric_c 20\nmetric_d 20\nmetric_e 999\n")) return nil case 3: - w.Write([]byte("metric_a 30\nmetric_b 30\nmetric_c 30\nmetric_d 30\n")) + _, _ = w.Write([]byte("metric_a 30\nmetric_b 30\nmetric_c 30\nmetric_d 30\n")) return nil case 4: cancel() @@ -1782,44 +1706,44 @@ func TestScrapeLoopRunCreatesStaleMarkersOnSampleLimit(t *testing.T) { // #2 - sample_limit exceeded - no samples appended, only 5 report series // #3 - success - 4 samples appended + 5 report series // #4 - scrape canceled - 4 StaleNaNs appended because of scrape error + 5 report series - require.Len(t, appender.resultFloats, (4+5)+5+(4+5)+(4+5), "Appended samples not as expected:\n%s", appender) + require.Len(t, appTest.ResultSamples, (4+5)+5+(4+5)+(4+5), "Appended samples not as expected:\n%s", appTest) // Expect first 4 samples to be metric_X [0-3]. for i := range 4 { - require.Equal(t, 10.0, appender.resultFloats[i].f, "Appended %d sample not as expected", i) + require.Equal(t, 10.0, appTest.ResultSamples[i].V, "Appended %d sample not as expected", i) } // Next 5 samples are report series [4-8]. // Next 5 samples are report series for the second scrape [9-13]. // Expect first 4 samples to be metric_X from the third scrape [14-17]. for i := 14; i <= 17; i++ { - require.Equal(t, 30.0, appender.resultFloats[i].f, "Appended %d sample not as expected", i) + require.Equal(t, 30.0, appTest.ResultSamples[i].V, "Appended %d sample not as expected", i) } // Next 5 samples are report series [18-22]. // Next 5 samples are report series [23-26]. for i := 23; i <= 26; i++ { - require.True(t, value.IsStaleNaN(appender.resultFloats[i].f), - "Appended second sample not as expected. Wanted: stale NaN Got: %x", math.Float64bits(appender.resultFloats[i].f)) + require.True(t, value.IsStaleNaN(appTest.ResultSamples[i].V), + "Appended second sample not as expected. Wanted: stale NaN Got: %x", math.Float64bits(appTest.ResultSamples[i].V)) } } func TestScrapeLoopCache(t *testing.T) { s := teststorage.New(t) - defer s.Close() + t.Cleanup(func() { _ = s.Close() }) - appender := &collectResultAppender{} - var ( - signal = make(chan struct{}, 1) - scraper = &testScraper{} - app = func(ctx context.Context) storage.Appender { appender.next = s.Appender(ctx); return appender } - ) + signal := make(chan struct{}, 1) - ctx, cancel := context.WithCancel(context.Background()) - // Decreasing the scrape interval could make the test fail, as multiple scrapes might be initiated at identical millisecond timestamps. - // See https://github.com/prometheus/prometheus/issues/12727. - // Since we're writing samples directly below we need to provide a protocol fallback. - sl := newBasicScrapeLoopWithFallback(t, ctx, scraper, app, 100*time.Millisecond, "text/plain") + ctx, cancel := context.WithCancel(t.Context()) + sl, scraper, appTest := newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.l = promslog.New(&promslog.Config{}) + sl.ctx = ctx + // Since we're writing samples directly below we need to provide a protocol fallback. + sl.fallbackScrapeProtocol = "text/plain" + // Decreasing the scrape interval could make the test fail, as multiple scrapes might be initiated at identical millisecond timestamps. + // See https://github.com/prometheus/prometheus/issues/12727. + sl.interval = 100 * time.Millisecond + }) + appTest.Next = s numScrapes := 0 - scraper.scrapeFunc = func(_ context.Context, w io.Writer) error { switch numScrapes { case 1, 2: @@ -1837,10 +1761,10 @@ func TestScrapeLoopCache(t *testing.T) { numScrapes++ switch numScrapes { case 1: - w.Write([]byte("metric_a 42\nmetric_b 43\n")) + _, _ = w.Write([]byte("metric_a 42\nmetric_b 43\n")) return nil case 3: - w.Write([]byte("metric_a 44\n")) + _, _ = w.Write([]byte("metric_a 44\n")) return nil case 4: cancel() @@ -1859,29 +1783,23 @@ func TestScrapeLoopCache(t *testing.T) { require.FailNow(t, "Scrape wasn't stopped.") } - // 1 successfully scraped sample, 1 stale marker after first fail, 5 report samples for - // each scrape successful or not. - require.Len(t, appender.resultFloats, 26, "Appended samples not as expected:\n%s", appender) + // 3 successfully scraped samples + // 3 stale marker after samples were missing. + // 4x 5 report samples for each scrape successful or not. + require.Len(t, appTest.ResultSamples, 26, "Appended samples not as expected:\n%s", appTest) } func TestScrapeLoopCacheMemoryExhaustionProtection(t *testing.T) { s := teststorage.New(t) - defer s.Close() + t.Cleanup(func() { _ = s.Close() }) - sapp := s.Appender(context.Background()) - - appender := &collectResultAppender{next: sapp} - var ( - signal = make(chan struct{}, 1) - scraper = &testScraper{} - app = func(context.Context) storage.Appender { return appender } - ) - - ctx, cancel := context.WithCancel(context.Background()) - sl := newBasicScrapeLoop(t, ctx, scraper, app, 10*time.Millisecond) + signal := make(chan struct{}, 1) + ctx, cancel := context.WithCancel(t.Context()) + sl, scraper, _ := newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.ctx = ctx + }) numScrapes := 0 - scraper.scrapeFunc = func(_ context.Context, w io.Writer) error { numScrapes++ if numScrapes < 5 { @@ -1889,7 +1807,7 @@ func TestScrapeLoopCacheMemoryExhaustionProtection(t *testing.T) { for i := range 500 { s = fmt.Sprintf("%smetric_%d_%d 42\n", s, i, numScrapes) } - w.Write([]byte(s + "&")) + _, _ = w.Write([]byte(s + "&")) } else { cancel() } @@ -1964,37 +1882,36 @@ func TestScrapeLoopAppend(t *testing.T) { } for _, test := range tests { - app := &collectResultAppender{} - discoveryLabels := &Target{ labels: labels.FromStrings(test.discoveryLabels...), } - sl := newBasicScrapeLoop(t, context.Background(), nil, func(context.Context) storage.Appender { return app }, 0) - sl.sampleMutator = func(l labels.Labels) labels.Labels { - return mutateSampleLabels(l, discoveryLabels, test.honorLabels, nil) - } - sl.reportSampleMutator = func(l labels.Labels) labels.Labels { - return mutateReportSampleLabels(l, discoveryLabels) - } + sl, _, appTest := newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.sampleMutator = func(l labels.Labels) labels.Labels { + return mutateSampleLabels(l, discoveryLabels, test.honorLabels, nil) + } + sl.reportSampleMutator = func(l labels.Labels) labels.Labels { + return mutateReportSampleLabels(l, discoveryLabels) + } + }) now := time.Now() - slApp := sl.appender(context.Background()) - _, _, _, err := sl.append(slApp, []byte(test.scrapeLabels), "text/plain", now) + slApp := sl.appender() + _, _, _, err := slApp.append([]byte(test.scrapeLabels), "text/plain", now) require.NoError(t, err) require.NoError(t, slApp.Commit()) - expected := []floatSample{ + expected := []sample{ { - metric: test.expLset, - t: timestamp.FromTime(now), - f: test.expValue, + L: test.expLset, + T: timestamp.FromTime(now), + V: test.expValue, }, } t.Logf("Test:%s", test.title) - requireEqual(t, expected, app.resultFloats) + requireEqual(t, expected, appTest.ResultSamples) } } @@ -2002,13 +1919,12 @@ func requireEqual(t *testing.T, expected, actual any, msgAndArgs ...any) { t.Helper() testutil.RequireEqualWithOptions(t, expected, actual, []cmp.Option{ - cmp.Comparer(equalFloatSamples), - cmp.AllowUnexported(histogramSample{}), + cmp.Comparer(func(a, b sample) bool { return a.Equal(b) }), // StaleNaN samples are generated by iterating over a map, which means that the order // of samples might be different on every test run. Sort series by label to avoid // test failures because of that. - cmpopts.SortSlices(func(a, b floatSample) int { - return labels.Compare(a.metric, b.metric) + cmpopts.SortSlices(func(a, b sample) int { + return labels.Compare(a.L, b.L) }), }, msgAndArgs...) @@ -2066,32 +1982,31 @@ func TestScrapeLoopAppendForConflictingPrefixedLabels(t *testing.T) { for name, tc := range testcases { t.Run(name, func(t *testing.T) { - app := &collectResultAppender{} - sl := newBasicScrapeLoop(t, context.Background(), nil, func(context.Context) storage.Appender { return app }, 0) - sl.sampleMutator = func(l labels.Labels) labels.Labels { - return mutateSampleLabels(l, &Target{labels: labels.FromStrings(tc.targetLabels...)}, false, nil) - } - slApp := sl.appender(context.Background()) - _, _, _, err := sl.append(slApp, []byte(tc.exposedLabels), "text/plain", time.Date(2000, 1, 1, 1, 0, 0, 0, time.UTC)) + sl, _, appTest := newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.sampleMutator = func(l labels.Labels) labels.Labels { + return mutateSampleLabels(l, &Target{labels: labels.FromStrings(tc.targetLabels...)}, false, nil) + } + }) + + slApp := sl.appender() + _, _, _, err := slApp.append([]byte(tc.exposedLabels), "text/plain", time.Date(2000, 1, 1, 1, 0, 0, 0, time.UTC)) require.NoError(t, err) require.NoError(t, slApp.Commit()) - requireEqual(t, []floatSample{ + requireEqual(t, []sample{ { - metric: labels.FromStrings(tc.expected...), - t: timestamp.FromTime(time.Date(2000, 1, 1, 1, 0, 0, 0, time.UTC)), - f: 0, + L: labels.FromStrings(tc.expected...), + T: timestamp.FromTime(time.Date(2000, 1, 1, 1, 0, 0, 0, time.UTC)), + V: 0, }, - }, app.resultFloats) + }, appTest.ResultSamples) }) } } func TestScrapeLoopAppendCacheEntryButErrNotFound(t *testing.T) { - // collectResultAppender's AddFast always returns ErrNotFound if we don't give it a next. - app := &collectResultAppender{} - sl := newBasicScrapeLoop(t, context.Background(), nil, func(context.Context) storage.Appender { return app }, 0) + sl, _, appTest := newTestScrapeLoop(t) fakeRef := storage.SeriesRef(1) expValue := float64(1) @@ -2101,7 +2016,8 @@ func TestScrapeLoopAppendCacheEntryButErrNotFound(t *testing.T) { require.NoError(t, warning) var lset labels.Labels - p.Next() + _, err := p.Next() + require.NoError(t, err) p.Labels(&lset) hash := lset.Hash() @@ -2109,34 +2025,33 @@ func TestScrapeLoopAppendCacheEntryButErrNotFound(t *testing.T) { sl.cache.addRef(metric, fakeRef, lset, hash) now := time.Now() - slApp := sl.appender(context.Background()) - _, _, _, err := sl.append(slApp, metric, "text/plain", now) + slApp := sl.appender() + _, _, _, err = slApp.append(metric, "text/plain", now) require.NoError(t, err) require.NoError(t, slApp.Commit()) - expected := []floatSample{ + expected := []sample{ { - metric: lset, - t: timestamp.FromTime(now), - f: expValue, + L: lset, + T: timestamp.FromTime(now), + V: expValue, }, } - require.Equal(t, expected, app.resultFloats) + require.Equal(t, expected, appTest.ResultSamples) } func TestScrapeLoopAppendSampleLimit(t *testing.T) { - resApp := &collectResultAppender{} - app := &limitAppender{Appender: resApp, limit: 1} - - sl := newBasicScrapeLoop(t, context.Background(), nil, func(context.Context) storage.Appender { return app }, 0) - sl.sampleMutator = func(l labels.Labels) labels.Labels { - if l.Has("deleteme") { - return labels.EmptyLabels() + sl, _, appTest := newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.sampleMutator = func(l labels.Labels) labels.Labels { + if l.Has("deleteme") { + return labels.EmptyLabels() + } + return l } - return l - } - sl.sampleLimit = app.limit + sl.sampleLimit = 1 // Same as limitAppender.limit + }) + // appTest.Prev = &limitAppender{AppenderV2: nopAppender{}, limit: 1} // Get the value of the Counter before performing the append. beforeMetric := dto.Metric{} @@ -2146,8 +2061,8 @@ func TestScrapeLoopAppendSampleLimit(t *testing.T) { beforeMetricValue := beforeMetric.GetCounter().GetValue() now := time.Now() - slApp := sl.appender(context.Background()) - total, added, seriesAdded, err := sl.append(app, []byte("metric_a 1\nmetric_b 1\nmetric_c 1\n"), "text/plain", now) + slApp := sl.appender() + total, added, seriesAdded, err := slApp.append([]byte("metric_a 1\nmetric_b 1\nmetric_c 1\n"), "text/plain", now) require.ErrorIs(t, err, errSampleLimit) require.NoError(t, slApp.Rollback()) require.Equal(t, 3, total) @@ -2160,23 +2075,23 @@ func TestScrapeLoopAppendSampleLimit(t *testing.T) { err = sl.metrics.targetScrapeSampleLimit.Write(&metric) require.NoError(t, err) - value := metric.GetCounter().GetValue() - change := value - beforeMetricValue + v := metric.GetCounter().GetValue() + change := v - beforeMetricValue require.Equal(t, 1.0, change, "Unexpected change of sample limit metric: %f", change) // And verify that we got the samples that fit under the limit. - want := []floatSample{ + want := []sample{ { - metric: labels.FromStrings(model.MetricNameLabel, "metric_a"), - t: timestamp.FromTime(now), - f: 1, + L: labels.FromStrings(model.MetricNameLabel, "metric_a"), + T: timestamp.FromTime(now), + V: 1, }, } - requireEqual(t, want, resApp.rolledbackFloats, "Appended samples not as expected:\n%s", appender) + requireEqual(t, want, appTest.RolledbackSamples, "Appended samples not as expected:\n%s", appTest) now = time.Now() - slApp = sl.appender(context.Background()) - total, added, seriesAdded, err = sl.append(slApp, []byte("metric_a 1\nmetric_b 1\nmetric_c{deleteme=\"yes\"} 1\nmetric_d 1\nmetric_e 1\nmetric_f 1\nmetric_g 1\nmetric_h{deleteme=\"yes\"} 1\nmetric_i{deleteme=\"yes\"} 1\n"), "text/plain", now) + slApp = sl.appender() + total, added, seriesAdded, err = slApp.append([]byte("metric_a 1\nmetric_b 1\nmetric_c{deleteme=\"yes\"} 1\nmetric_d 1\nmetric_e 1\nmetric_f 1\nmetric_g 1\nmetric_h{deleteme=\"yes\"} 1\nmetric_i{deleteme=\"yes\"} 1\n"), "text/plain", now) require.ErrorIs(t, err, errSampleLimit) require.NoError(t, slApp.Rollback()) require.Equal(t, 9, total) @@ -2185,17 +2100,18 @@ func TestScrapeLoopAppendSampleLimit(t *testing.T) { } func TestScrapeLoop_HistogramBucketLimit(t *testing.T) { - resApp := &collectResultAppender{} - app := &bucketLimitAppender{Appender: resApp, limit: 2} - - sl := newBasicScrapeLoop(t, context.Background(), nil, func(context.Context) storage.Appender { return app }, 0) - sl.enableNativeHistogramScraping = true - sl.sampleMutator = func(l labels.Labels) labels.Labels { - if l.Has("deleteme") { - return labels.EmptyLabels() + sl, _, _ := newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.enableNativeHistogramScraping = true + sl.sampleMutator = func(l labels.Labels) labels.Labels { + if l.Has("deleteme") { + return labels.EmptyLabels() + } + return l } - return l - } + }) + + // appTest.Prev = &bucketLimitAppender{AppenderV2: nopAppender{}, limit: 2} + slApp := sl.appender() metric := dto.Metric{} err := sl.metrics.targetScrapeNativeHistogramBucketLimit.Write(&metric) @@ -2214,7 +2130,7 @@ func TestScrapeLoop_HistogramBucketLimit(t *testing.T) { []string{"size"}, ) registry := prometheus.NewRegistry() - registry.Register(nativeHistogram) + require.NoError(t, registry.Register(nativeHistogram)) nativeHistogram.WithLabelValues("S").Observe(1.0) nativeHistogram.WithLabelValues("M").Observe(1.0) nativeHistogram.WithLabelValues("L").Observe(1.0) @@ -2230,7 +2146,7 @@ func TestScrapeLoop_HistogramBucketLimit(t *testing.T) { require.NoError(t, err) now := time.Now() - total, added, seriesAdded, err := sl.append(app, msg, "application/vnd.google.protobuf", now) + total, added, seriesAdded, err := slApp.append(msg, "application/vnd.google.protobuf", now) require.NoError(t, err) require.Equal(t, 3, total) require.Equal(t, 3, added) @@ -2253,7 +2169,7 @@ func TestScrapeLoop_HistogramBucketLimit(t *testing.T) { require.NoError(t, err) now = time.Now() - total, added, seriesAdded, err = sl.append(app, msg, "application/vnd.google.protobuf", now) + total, added, seriesAdded, err = slApp.append(msg, "application/vnd.google.protobuf", now) require.NoError(t, err) require.Equal(t, 3, total) require.Equal(t, 3, added) @@ -2276,11 +2192,11 @@ func TestScrapeLoop_HistogramBucketLimit(t *testing.T) { require.NoError(t, err) now = time.Now() - total, added, seriesAdded, err = sl.append(app, msg, "application/vnd.google.protobuf", now) + total, added, seriesAdded, err = slApp.append(msg, "application/vnd.google.protobuf", now) if !errors.Is(err, errBucketLimit) { t.Fatalf("Did not see expected histogram bucket limit error: %s", err) } - require.NoError(t, app.Rollback()) + require.NoError(t, slApp.Rollback()) require.Equal(t, 3, total) require.Equal(t, 3, added) require.Equal(t, 0, seriesAdded) @@ -2296,148 +2212,144 @@ func TestScrapeLoop_ChangingMetricString(t *testing.T) { // IDs when the string representation of a metric changes across a scrape. Thus // we use a real storage appender here. s := teststorage.New(t) - defer s.Close() + t.Cleanup(func() { _ = s.Close() }) - capp := &collectResultAppender{} - sl := newBasicScrapeLoop(t, context.Background(), nil, func(context.Context) storage.Appender { return capp }, 0) + sl, _, appTest := newTestScrapeLoop(t) now := time.Now() - slApp := sl.appender(context.Background()) - _, _, _, err := sl.append(slApp, []byte(`metric_a{a="1",b="1"} 1`), "text/plain", now) + slApp := sl.appender() + _, _, _, err := slApp.append([]byte(`metric_a{a="1",b="1"} 1`), "text/plain", now) require.NoError(t, err) require.NoError(t, slApp.Commit()) - slApp = sl.appender(context.Background()) - _, _, _, err = sl.append(slApp, []byte(`metric_a{b="1",a="1"} 2`), "text/plain", now.Add(time.Minute)) + slApp = sl.appender() + _, _, _, err = slApp.append([]byte(`metric_a{b="1",a="1"} 2`), "text/plain", now.Add(time.Minute)) require.NoError(t, err) require.NoError(t, slApp.Commit()) - want := []floatSample{ + want := []sample{ { - metric: labels.FromStrings("__name__", "metric_a", "a", "1", "b", "1"), - t: timestamp.FromTime(now), - f: 1, + L: labels.FromStrings("__name__", "metric_a", "a", "1", "b", "1"), + T: timestamp.FromTime(now), + V: 1, }, { - metric: labels.FromStrings("__name__", "metric_a", "a", "1", "b", "1"), - t: timestamp.FromTime(now.Add(time.Minute)), - f: 2, + L: labels.FromStrings("__name__", "metric_a", "a", "1", "b", "1"), + T: timestamp.FromTime(now.Add(time.Minute)), + V: 2, }, } - require.Equal(t, want, capp.resultFloats, "Appended samples not as expected:\n%s", appender) + require.Equal(t, want, appTest.ResultSamples, "Appended samples not as expected:\n%s", appTest) } func TestScrapeLoopAppendFailsWithNoContentType(t *testing.T) { - app := &collectResultAppender{} - - // Explicitly setting the lack of fallback protocol here to make it obvious. - sl := newBasicScrapeLoopWithFallback(t, context.Background(), nil, func(context.Context) storage.Appender { return app }, 0, "") + sl, _, _ := newTestScrapeLoop(t, func(sl *scrapeLoop) { + // Explicitly setting the lack of fallback protocol here to make it obvious. + sl.fallbackScrapeProtocol = "" + }) now := time.Now() - slApp := sl.appender(context.Background()) - _, _, _, err := sl.append(slApp, []byte("metric_a 1\n"), "", now) + slApp := sl.appender() + _, _, _, err := slApp.append([]byte("metric_a 1\n"), "", now) // We expect the appropriate error. require.ErrorContains(t, err, "non-compliant scrape target sending blank Content-Type and no fallback_scrape_protocol specified for target", "Expected \"non-compliant scrape\" error but got: %s", err) } +// TestScrapeLoopAppendEmptyWithNoContentType ensures we there are no errors when we get a blank scrape or just want to append a stale marker. func TestScrapeLoopAppendEmptyWithNoContentType(t *testing.T) { - // This test ensures we there are no errors when we get a blank scrape or just want to append a stale marker. - app := &collectResultAppender{} - - // Explicitly setting the lack of fallback protocol here to make it obvious. - sl := newBasicScrapeLoopWithFallback(t, context.Background(), nil, func(context.Context) storage.Appender { return app }, 0, "") + sl, _, _ := newTestScrapeLoop(t, func(sl *scrapeLoop) { + // Explicitly setting the lack of fallback protocol here to make it obvious. + sl.fallbackScrapeProtocol = "" + }) now := time.Now() - slApp := sl.appender(context.Background()) - _, _, _, err := sl.append(slApp, []byte(""), "", now) + slApp := sl.appender() + _, _, _, err := slApp.append([]byte(""), "", now) require.NoError(t, err) require.NoError(t, slApp.Commit()) } func TestScrapeLoopAppendStaleness(t *testing.T) { - app := &collectResultAppender{} - - sl := newBasicScrapeLoop(t, context.Background(), nil, func(context.Context) storage.Appender { return app }, 0) + sl, _, appTest := newTestScrapeLoop(t) now := time.Now() - slApp := sl.appender(context.Background()) - _, _, _, err := sl.append(slApp, []byte("metric_a 1\n"), "text/plain", now) + slApp := sl.appender() + _, _, _, err := slApp.append([]byte("metric_a 1\n"), "text/plain", now) require.NoError(t, err) require.NoError(t, slApp.Commit()) - slApp = sl.appender(context.Background()) - _, _, _, err = sl.append(slApp, []byte(""), "", now.Add(time.Second)) + slApp = sl.appender() + _, _, _, err = slApp.append([]byte(""), "", now.Add(time.Second)) require.NoError(t, err) require.NoError(t, slApp.Commit()) - want := []floatSample{ + want := []sample{ { - metric: labels.FromStrings(model.MetricNameLabel, "metric_a"), - t: timestamp.FromTime(now), - f: 1, + L: labels.FromStrings(model.MetricNameLabel, "metric_a"), + T: timestamp.FromTime(now), + V: 1, }, { - metric: labels.FromStrings(model.MetricNameLabel, "metric_a"), - t: timestamp.FromTime(now.Add(time.Second)), - f: math.Float64frombits(value.StaleNaN), + L: labels.FromStrings(model.MetricNameLabel, "metric_a"), + T: timestamp.FromTime(now.Add(time.Second)), + V: math.Float64frombits(value.StaleNaN), }, } - requireEqual(t, want, app.resultFloats, "Appended samples not as expected:\n%s", appender) + requireEqual(t, want, appTest.ResultSamples, "Appended samples not as expected:\n%s", appTest) } func TestScrapeLoopAppendNoStalenessIfTimestamp(t *testing.T) { - app := &collectResultAppender{} - sl := newBasicScrapeLoop(t, context.Background(), nil, func(context.Context) storage.Appender { return app }, 0) + sl, _, appTest := newTestScrapeLoop(t) now := time.Now() - slApp := sl.appender(context.Background()) - _, _, _, err := sl.append(slApp, []byte("metric_a 1 1000\n"), "text/plain", now) + slApp := sl.appender() + _, _, _, err := slApp.append([]byte("metric_a 1 1000\n"), "text/plain", now) require.NoError(t, err) require.NoError(t, slApp.Commit()) - slApp = sl.appender(context.Background()) - _, _, _, err = sl.append(slApp, []byte(""), "", now.Add(time.Second)) + slApp = sl.appender() + _, _, _, err = slApp.append([]byte(""), "", now.Add(time.Second)) require.NoError(t, err) require.NoError(t, slApp.Commit()) - want := []floatSample{ + want := []sample{ { - metric: labels.FromStrings(model.MetricNameLabel, "metric_a"), - t: 1000, - f: 1, + L: labels.FromStrings(model.MetricNameLabel, "metric_a"), + T: 1000, + V: 1, }, } - require.Equal(t, want, app.resultFloats, "Appended samples not as expected:\n%s", appender) + require.Equal(t, want, appTest.ResultSamples, "Appended samples not as expected:\n%s", appTest) } func TestScrapeLoopAppendStalenessIfTrackTimestampStaleness(t *testing.T) { - app := &collectResultAppender{} - sl := newBasicScrapeLoop(t, context.Background(), nil, func(context.Context) storage.Appender { return app }, 0) - sl.trackTimestampsStaleness = true + sl, _, appTest := newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.trackTimestampsStaleness = true + }) now := time.Now() - slApp := sl.appender(context.Background()) - _, _, _, err := sl.append(slApp, []byte("metric_a 1 1000\n"), "text/plain", now) + slApp := sl.appender() + _, _, _, err := slApp.append([]byte("metric_a 1 1000\n"), "text/plain", now) require.NoError(t, err) require.NoError(t, slApp.Commit()) - slApp = sl.appender(context.Background()) - _, _, _, err = sl.append(slApp, []byte(""), "", now.Add(time.Second)) + slApp = sl.appender() + _, _, _, err = slApp.append([]byte(""), "", now.Add(time.Second)) require.NoError(t, err) require.NoError(t, slApp.Commit()) - want := []floatSample{ + want := []sample{ { - metric: labels.FromStrings(model.MetricNameLabel, "metric_a"), - t: 1000, - f: 1, + L: labels.FromStrings(model.MetricNameLabel, "metric_a"), + T: 1000, + V: 1, }, { - metric: labels.FromStrings(model.MetricNameLabel, "metric_a"), - t: timestamp.FromTime(now.Add(time.Second)), - f: math.Float64frombits(value.StaleNaN), + L: labels.FromStrings(model.MetricNameLabel, "metric_a"), + T: timestamp.FromTime(now.Add(time.Second)), + V: math.Float64frombits(value.StaleNaN), }, } - requireEqual(t, want, app.resultFloats, "Appended samples not as expected:\n%s", appender) + requireEqual(t, want, appTest.ResultSamples, "Appended samples not as expected:\n%s", appTest) } func TestScrapeLoopAppendExemplar(t *testing.T) { @@ -2448,18 +2360,16 @@ func TestScrapeLoopAppendExemplar(t *testing.T) { scrapeText string contentType string discoveryLabels []string - floats []floatSample - histograms []histogramSample - exemplars []exemplar.Exemplar + samples []sample }{ { title: "Metric without exemplars", scrapeText: "metric_total{n=\"1\"} 0\n# EOF", contentType: "application/openmetrics-text", discoveryLabels: []string{"n", "2"}, - floats: []floatSample{{ - metric: labels.FromStrings("__name__", "metric_total", "exported_n", "1", "n", "2"), - f: 0, + samples: []sample{{ + L: labels.FromStrings("__name__", "metric_total", "exported_n", "1", "n", "2"), + V: 0, }}, }, { @@ -2467,26 +2377,24 @@ func TestScrapeLoopAppendExemplar(t *testing.T) { scrapeText: "metric_total{n=\"1\"} 0 # {a=\"abc\"} 1.0\n# EOF", contentType: "application/openmetrics-text", discoveryLabels: []string{"n", "2"}, - floats: []floatSample{{ - metric: labels.FromStrings("__name__", "metric_total", "exported_n", "1", "n", "2"), - f: 0, + samples: []sample{{ + L: labels.FromStrings("__name__", "metric_total", "exported_n", "1", "n", "2"), + V: 0, + ES: []exemplar.Exemplar{ + {Labels: labels.FromStrings("a", "abc"), Value: 1}, + }, }}, - exemplars: []exemplar.Exemplar{ - {Labels: labels.FromStrings("a", "abc"), Value: 1}, - }, }, { title: "Metric with exemplars and TS", scrapeText: "metric_total{n=\"1\"} 0 # {a=\"abc\"} 1.0 10000\n# EOF", contentType: "application/openmetrics-text", discoveryLabels: []string{"n", "2"}, - floats: []floatSample{{ - metric: labels.FromStrings("__name__", "metric_total", "exported_n", "1", "n", "2"), - f: 0, + samples: []sample{{ + L: labels.FromStrings("__name__", "metric_total", "exported_n", "1", "n", "2"), + V: 0, + ES: []exemplar.Exemplar{{Labels: labels.FromStrings("a", "abc"), Value: 1, Ts: 10000000, HasTs: true}}, }}, - exemplars: []exemplar.Exemplar{ - {Labels: labels.FromStrings("a", "abc"), Value: 1, Ts: 10000000, HasTs: true}, - }, }, { title: "Two metrics and exemplars", @@ -2494,17 +2402,15 @@ func TestScrapeLoopAppendExemplar(t *testing.T) { metric_total{n="2"} 2 # {t="2"} 2.0 20000 # EOF`, contentType: "application/openmetrics-text", - floats: []floatSample{{ - metric: labels.FromStrings("__name__", "metric_total", "n", "1"), - f: 1, + samples: []sample{{ + L: labels.FromStrings("__name__", "metric_total", "n", "1"), + V: 1, + ES: []exemplar.Exemplar{{Labels: labels.FromStrings("t", "1"), Value: 1, Ts: 10000000, HasTs: true}}, }, { - metric: labels.FromStrings("__name__", "metric_total", "n", "2"), - f: 2, + L: labels.FromStrings("__name__", "metric_total", "n", "2"), + V: 2, + ES: []exemplar.Exemplar{{Labels: labels.FromStrings("t", "2"), Value: 2, Ts: 20000000, HasTs: true}}, }}, - exemplars: []exemplar.Exemplar{ - {Labels: labels.FromStrings("t", "1"), Value: 1, Ts: 10000000, HasTs: true}, - {Labels: labels.FromStrings("t", "2"), Value: 2, Ts: 20000000, HasTs: true}, - }, }, { title: "Native histogram with three exemplars from classic buckets", @@ -2596,10 +2502,10 @@ metric: < `, contentType: "application/vnd.google.protobuf", - histograms: []histogramSample{{ - t: 1234568, - metric: labels.FromStrings("__name__", "test_histogram"), - h: &histogram.Histogram{ + samples: []sample{{ + T: 1234568, + L: labels.FromStrings("__name__", "test_histogram"), + H: &histogram.Histogram{ Count: 175, ZeroCount: 2, Sum: 0.0008280461746287094, @@ -2616,12 +2522,12 @@ metric: < PositiveBuckets: []int64{1, 2, -1, -1}, NegativeBuckets: []int64{1, 3, -2, -1, 1}, }, + ES: []exemplar.Exemplar{ + // Native histogram exemplars are arranged by timestamp, and those with missing timestamps are dropped. + {Labels: labels.FromStrings("dummyID", "58215"), Value: -0.00019, Ts: 1625851055146, HasTs: true}, + {Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00039, Ts: 1625851155146, HasTs: true}, + }, }}, - exemplars: []exemplar.Exemplar{ - // Native histogram exemplars are arranged by timestamp, and those with missing timestamps are dropped. - {Labels: labels.FromStrings("dummyID", "58215"), Value: -0.00019, Ts: 1625851055146, HasTs: true}, - {Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00039, Ts: 1625851155146, HasTs: true}, - }, }, { title: "Native histogram with three exemplars scraped as classic histogram", @@ -2714,46 +2620,46 @@ metric: < `, alwaysScrapeClassicHist: true, contentType: "application/vnd.google.protobuf", - floats: []floatSample{ - {metric: labels.FromStrings("__name__", "test_histogram_count"), t: 1234568, f: 175}, - {metric: labels.FromStrings("__name__", "test_histogram_sum"), t: 1234568, f: 0.0008280461746287094}, - {metric: labels.FromStrings("__name__", "test_histogram_bucket", "le", "-0.0004899999999999998"), t: 1234568, f: 2}, - {metric: labels.FromStrings("__name__", "test_histogram_bucket", "le", "-0.0003899999999999998"), t: 1234568, f: 4}, - {metric: labels.FromStrings("__name__", "test_histogram_bucket", "le", "-0.0002899999999999998"), t: 1234568, f: 16}, - {metric: labels.FromStrings("__name__", "test_histogram_bucket", "le", "-0.0001899999999999998"), t: 1234568, f: 32}, - {metric: labels.FromStrings("__name__", "test_histogram_bucket", "le", "+Inf"), t: 1234568, f: 175}, - }, - histograms: []histogramSample{{ - t: 1234568, - metric: labels.FromStrings("__name__", "test_histogram"), - h: &histogram.Histogram{ - Count: 175, - ZeroCount: 2, - Sum: 0.0008280461746287094, - ZeroThreshold: 2.938735877055719e-39, - Schema: 3, - PositiveSpans: []histogram.Span{ - {Offset: -161, Length: 1}, - {Offset: 8, Length: 3}, + samples: []sample{ + {L: labels.FromStrings("__name__", "test_histogram_count"), T: 1234568, V: 175}, + {L: labels.FromStrings("__name__", "test_histogram_sum"), T: 1234568, V: 0.0008280461746287094}, + {L: labels.FromStrings("__name__", "test_histogram_bucket", "le", "-0.0004899999999999998"), T: 1234568, V: 2}, + {L: labels.FromStrings("__name__", "test_histogram_bucket", "le", "-0.0003899999999999998"), T: 1234568, V: 4}, + {L: labels.FromStrings("__name__", "test_histogram_bucket", "le", "-0.0002899999999999998"), T: 1234568, V: 16}, + {L: labels.FromStrings("__name__", "test_histogram_bucket", "le", "-0.0001899999999999998"), T: 1234568, V: 32}, + {L: labels.FromStrings("__name__", "test_histogram_bucket", "le", "+Inf"), T: 1234568, V: 175}, + { + T: 1234568, + L: labels.FromStrings("__name__", "test_histogram"), + H: &histogram.Histogram{ + Count: 175, + ZeroCount: 2, + Sum: 0.0008280461746287094, + ZeroThreshold: 2.938735877055719e-39, + Schema: 3, + PositiveSpans: []histogram.Span{ + {Offset: -161, Length: 1}, + {Offset: 8, Length: 3}, + }, + NegativeSpans: []histogram.Span{ + {Offset: -162, Length: 1}, + {Offset: 23, Length: 4}, + }, + PositiveBuckets: []int64{1, 2, -1, -1}, + NegativeBuckets: []int64{1, 3, -2, -1, 1}, }, - NegativeSpans: []histogram.Span{ - {Offset: -162, Length: 1}, - {Offset: 23, Length: 4}, + ES: []exemplar.Exemplar{ + // Native histogram one is arranged by timestamp. + // Exemplars with missing timestamps are dropped for native histograms. + {Labels: labels.FromStrings("dummyID", "58215"), Value: -0.00019, Ts: 1625851055146, HasTs: true}, + {Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00039, Ts: 1625851155146, HasTs: true}, + // Classic histogram one is in order of appearance. + // Exemplars with missing timestamps are supported for classic histograms. + {Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00039, Ts: 1625851155146, HasTs: true}, + {Labels: labels.FromStrings("dummyID", "5617"), Value: -0.00029, Ts: 1234568, HasTs: false}, + {Labels: labels.FromStrings("dummyID", "58215"), Value: -0.00019, Ts: 1625851055146, HasTs: true}, }, - PositiveBuckets: []int64{1, 2, -1, -1}, - NegativeBuckets: []int64{1, 3, -2, -1, 1}, }, - }}, - exemplars: []exemplar.Exemplar{ - // Native histogram one is arranged by timestamp. - // Exemplars with missing timestamps are dropped for native histograms. - {Labels: labels.FromStrings("dummyID", "58215"), Value: -0.00019, Ts: 1625851055146, HasTs: true}, - {Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00039, Ts: 1625851155146, HasTs: true}, - // Classic histogram one is in order of appearance. - // Exemplars with missing timestamps are supported for classic histograms. - {Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00039, Ts: 1625851155146, HasTs: true}, - {Labels: labels.FromStrings("dummyID", "5617"), Value: -0.00029, Ts: 1234568, HasTs: false}, - {Labels: labels.FromStrings("dummyID", "58215"), Value: -0.00019, Ts: 1625851055146, HasTs: true}, }, }, { @@ -2829,10 +2735,10 @@ metric: < > `, - histograms: []histogramSample{{ - t: 1234568, - metric: labels.FromStrings("__name__", "test_histogram"), - h: &histogram.Histogram{ + samples: []sample{{ + T: 1234568, + L: labels.FromStrings("__name__", "test_histogram"), + H: &histogram.Histogram{ Count: 175, ZeroCount: 2, Sum: 0.0008280461746287094, @@ -2849,12 +2755,12 @@ metric: < PositiveBuckets: []int64{1, 2, -1, -1}, NegativeBuckets: []int64{1, 3, -2, -1, 1}, }, + ES: []exemplar.Exemplar{ + // Exemplars with missing timestamps are dropped for native histograms. + {Labels: labels.FromStrings("dummyID", "58242"), Value: -0.00019, Ts: 1625851055146, HasTs: true}, + {Labels: labels.FromStrings("dummyID", "59732"), Value: -0.00039, Ts: 1625851155146, HasTs: true}, + }, }}, - exemplars: []exemplar.Exemplar{ - // Exemplars with missing timestamps are dropped for native histograms. - {Labels: labels.FromStrings("dummyID", "58242"), Value: -0.00019, Ts: 1625851055146, HasTs: true}, - {Labels: labels.FromStrings("dummyID", "59732"), Value: -0.00039, Ts: 1625851155146, HasTs: true}, - }, }, { title: "Native histogram with exemplars but ingestion disabled", @@ -2929,45 +2835,45 @@ metric: < > `, - floats: []floatSample{ - {metric: labels.FromStrings("__name__", "test_histogram_count"), t: 1234568, f: 175}, - {metric: labels.FromStrings("__name__", "test_histogram_sum"), t: 1234568, f: 0.0008280461746287094}, - {metric: labels.FromStrings("__name__", "test_histogram_bucket", "le", "+Inf"), t: 1234568, f: 175}, + samples: []sample{ + {L: labels.FromStrings("__name__", "test_histogram_count"), T: 1234568, V: 175}, + {L: labels.FromStrings("__name__", "test_histogram_sum"), T: 1234568, V: 0.0008280461746287094}, + {L: labels.FromStrings("__name__", "test_histogram_bucket", "le", "+Inf"), T: 1234568, V: 175}, }, }, } for _, test := range tests { t.Run(test.title, func(t *testing.T) { - app := &collectResultAppender{} - discoveryLabels := &Target{ labels: labels.FromStrings(test.discoveryLabels...), } - sl := newBasicScrapeLoop(t, context.Background(), nil, func(context.Context) storage.Appender { return app }, 0) - sl.enableNativeHistogramScraping = test.enableNativeHistogramsIngestion - sl.sampleMutator = func(l labels.Labels) labels.Labels { - return mutateSampleLabels(l, discoveryLabels, false, nil) - } - sl.reportSampleMutator = func(l labels.Labels) labels.Labels { - return mutateReportSampleLabels(l, discoveryLabels) - } - sl.alwaysScrapeClassicHist = test.alwaysScrapeClassicHist + sl, _, appTest := newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.enableNativeHistogramScraping = test.enableNativeHistogramsIngestion + sl.sampleMutator = func(l labels.Labels) labels.Labels { + return mutateSampleLabels(l, discoveryLabels, false, nil) + } + sl.reportSampleMutator = func(l labels.Labels) labels.Labels { + return mutateReportSampleLabels(l, discoveryLabels) + } + sl.alwaysScrapeClassicHist = test.alwaysScrapeClassicHist + }) + slApp := sl.appender() now := time.Now() - for i := range test.floats { - if test.floats[i].t != 0 { + for i := range test.samples { + if test.samples[i].T != 0 { continue } - test.floats[i].t = timestamp.FromTime(now) - } + test.samples[i].T = timestamp.FromTime(now) - // We need to set the timestamp for expected exemplars that does not have a timestamp. - for i := range test.exemplars { - if test.exemplars[i].Ts == 0 { - test.exemplars[i].Ts = timestamp.FromTime(now) + // We need to set the timestamp for expected exemplars that does not have a timestamp. + for j := range test.samples[i].ES { + if test.samples[i].ES[j].Ts == 0 { + test.samples[i].ES[j].Ts = timestamp.FromTime(now) + } } } @@ -2978,12 +2884,10 @@ metric: < buf.WriteString(test.scrapeText) } - _, _, _, err := sl.append(app, buf.Bytes(), test.contentType, now) + _, _, _, err := slApp.append(buf.Bytes(), test.contentType, now) require.NoError(t, err) - require.NoError(t, app.Commit()) - requireEqual(t, test.floats, app.resultFloats) - requireEqual(t, test.histograms, app.resultHistograms) - requireEqual(t, test.exemplars, app.resultExemplars) + require.NoError(t, slApp.Commit()) + requireEqual(t, test.samples, appTest.ResultSamples) }) } } @@ -3012,12 +2916,12 @@ func TestScrapeLoopAppendExemplarSeries(t *testing.T) { scrapeText := []string{`metric_total{n="1"} 1 # {t="1"} 1.0 10000 # EOF`, `metric_total{n="1"} 2 # {t="2"} 2.0 20000 # EOF`} - samples := []floatSample{{ - metric: labels.FromStrings("__name__", "metric_total", "n", "1"), - f: 1, + samples := []sample{{ + L: labels.FromStrings("__name__", "metric_total", "n", "1"), + V: 1, }, { - metric: labels.FromStrings("__name__", "metric_total", "n", "1"), - f: 2, + L: labels.FromStrings("__name__", "metric_total", "n", "1"), + V: 2, }} exemplars := []exemplar.Exemplar{ {Labels: labels.FromStrings("t", "1"), Value: 1, Ts: 10000000, HasTs: true}, @@ -3027,21 +2931,22 @@ func TestScrapeLoopAppendExemplarSeries(t *testing.T) { labels: labels.FromStrings(), } - app := &collectResultAppender{} + sl, _, appTest := newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.sampleMutator = func(l labels.Labels) labels.Labels { + return mutateSampleLabels(l, discoveryLabels, false, nil) + } + sl.reportSampleMutator = func(l labels.Labels) labels.Labels { + return mutateReportSampleLabels(l, discoveryLabels) + } + }) - sl := newBasicScrapeLoop(t, context.Background(), nil, func(context.Context) storage.Appender { return app }, 0) - sl.sampleMutator = func(l labels.Labels) labels.Labels { - return mutateSampleLabels(l, discoveryLabels, false, nil) - } - sl.reportSampleMutator = func(l labels.Labels) labels.Labels { - return mutateReportSampleLabels(l, discoveryLabels) - } + slApp := sl.appender() now := time.Now() for i := range samples { ts := now.Add(time.Second * time.Duration(i)) - samples[i].t = timestamp.FromTime(ts) + samples[i].T = timestamp.FromTime(ts) } // We need to set the timestamp for expected exemplars that does not have a timestamp. @@ -3053,60 +2958,49 @@ func TestScrapeLoopAppendExemplarSeries(t *testing.T) { } for i, st := range scrapeText { - _, _, _, err := sl.append(app, []byte(st), "application/openmetrics-text", timestamp.Time(samples[i].t)) + _, _, _, err := slApp.append([]byte(st), "application/openmetrics-text", timestamp.Time(samples[i].T)) require.NoError(t, err) - require.NoError(t, app.Commit()) + require.NoError(t, slApp.Commit()) } - requireEqual(t, samples, app.resultFloats) - requireEqual(t, exemplars, app.resultExemplars) + requireEqual(t, samples, appTest.ResultSamples) } func TestScrapeLoopRunReportsTargetDownOnScrapeError(t *testing.T) { - var ( - scraper = &testScraper{} - appender = &collectResultAppender{} - app = func(context.Context) storage.Appender { return appender } - ) - - ctx, cancel := context.WithCancel(context.Background()) - sl := newBasicScrapeLoop(t, ctx, scraper, app, 10*time.Millisecond) - + ctx, cancel := context.WithCancel(t.Context()) + sl, scraper, appTest := newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.ctx = ctx + }) scraper.scrapeFunc = func(context.Context, io.Writer) error { cancel() return errors.New("scrape failed") } sl.run(nil) - require.Equal(t, 0.0, appender.resultFloats[0].f, "bad 'up' value") + require.Equal(t, 0.0, appTest.ResultSamples[0].V, "bad 'up' value") } func TestScrapeLoopRunReportsTargetDownOnInvalidUTF8(t *testing.T) { - var ( - scraper = &testScraper{} - appender = &collectResultAppender{} - app = func(context.Context) storage.Appender { return appender } - ) - - ctx, cancel := context.WithCancel(context.Background()) - sl := newBasicScrapeLoop(t, ctx, scraper, app, 10*time.Millisecond) - + ctx, cancel := context.WithCancel(t.Context()) + sl, scraper, appTest := newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.ctx = ctx + }) scraper.scrapeFunc = func(_ context.Context, w io.Writer) error { cancel() - w.Write([]byte("a{l=\"\xff\"} 1\n")) + _, _ = w.Write([]byte("a{l=\"\xff\"} 1\n")) return nil } sl.run(nil) - require.Equal(t, 0.0, appender.resultFloats[0].f, "bad 'up' value") + require.Equal(t, 0.0, appTest.ResultSamples[0].V, "bad 'up' value") } type errorAppender struct { - collectResultAppender + storage.AppenderV2 } -func (app *errorAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64, v float64) (storage.SeriesRef, error) { - switch lset.Get(model.MetricNameLabel) { +func (app *errorAppender) Append(ref storage.SeriesRef, ls labels.Labels, st, t int64, v float64, h *histogram.Histogram, fh *histogram.FloatHistogram, opts storage.AOptions) (storage.SeriesRef, error) { + switch ls.Get(model.MetricNameLabel) { case "out_of_order": return 0, storage.ErrOutOfOrderSample case "amend": @@ -3114,48 +3008,43 @@ func (app *errorAppender) Append(ref storage.SeriesRef, lset labels.Labels, t in case "out_of_bounds": return 0, storage.ErrOutOfBounds default: - return app.collectResultAppender.Append(ref, lset, t, v) + return app.AppenderV2.Append(ref, ls, st, t, v, h, fh, opts) } } func TestScrapeLoopAppendGracefullyIfAmendOrOutOfOrderOrOutOfBounds(t *testing.T) { - app := &errorAppender{} - sl := newBasicScrapeLoop(t, context.Background(), nil, func(context.Context) storage.Appender { return app }, 0) + sl, _, appTest := newTestScrapeLoop(t) + // appTest.Prev = &errorAppender{AppenderV2: nopAppender{}} now := time.Unix(1, 0) - slApp := sl.appender(context.Background()) - total, added, seriesAdded, err := sl.append(slApp, []byte("out_of_order 1\namend 1\nnormal 1\nout_of_bounds 1\n"), "text/plain", now) + slApp := sl.appender() + total, added, seriesAdded, err := slApp.append([]byte("out_of_order 1\namend 1\nnormal 1\nout_of_bounds 1\n"), "text/plain", now) require.NoError(t, err) require.NoError(t, slApp.Commit()) - want := []floatSample{ + want := []sample{ { - metric: labels.FromStrings(model.MetricNameLabel, "normal"), - t: timestamp.FromTime(now), - f: 1, + L: labels.FromStrings(model.MetricNameLabel, "normal"), + T: timestamp.FromTime(now), + V: 1, }, } - requireEqual(t, want, app.resultFloats, "Appended samples not as expected:\n%s", appender) + requireEqual(t, want, appTest.ResultSamples, "Appended samples not as expected:\n%s", appTest) require.Equal(t, 4, total) require.Equal(t, 4, added) require.Equal(t, 1, seriesAdded) } func TestScrapeLoopOutOfBoundsTimeError(t *testing.T) { - app := &collectResultAppender{} - sl := newBasicScrapeLoop(t, context.Background(), nil, - func(context.Context) storage.Appender { - return &timeLimitAppender{ - Appender: app, - maxTime: timestamp.FromTime(time.Now().Add(10 * time.Minute)), - } - }, - 0, - ) + sl, _, _ := newTestScrapeLoop(t) + //appTest.Prev = &timeLimitAppender{ + // AppenderV2: nopAppender{}, + // maxTime: timestamp.FromTime(time.Now().Add(10 * time.Minute)), + //} now := time.Now().Add(20 * time.Minute) - slApp := sl.appender(context.Background()) - total, added, seriesAdded, err := sl.append(slApp, []byte("normal 1\n"), "text/plain", now) + slApp := sl.appender() + total, added, seriesAdded, err := slApp.append([]byte("normal 1\n"), "text/plain", now) require.NoError(t, err) require.NoError(t, slApp.Commit()) require.Equal(t, 1, total) @@ -3252,7 +3141,7 @@ func TestRequestTraceparentHeader(t *testing.T) { resp, err := ts.scrape(context.Background()) require.NoError(t, err) require.NotNil(t, resp) - defer resp.Body.Close() + t.Cleanup(func() { _ = resp.Body.Close() }) } func TestTargetScraperScrapeOK(t *testing.T) { @@ -3299,7 +3188,7 @@ func TestTargetScraperScrapeOK(t *testing.T) { } else { w.Header().Set("Content-Type", `text/plain; version=0.0.4`) } - w.Write([]byte("metric_a 1\nmetric_b 2\n")) + _, _ = w.Write([]byte("metric_a 1\nmetric_b 2\n")) }), ) defer server.Close() @@ -3476,11 +3365,11 @@ func TestTargetScraperBodySizeLimit(t *testing.T) { if gzipResponse { w.Header().Set("Content-Encoding", "gzip") gw := gzip.NewWriter(w) - defer gw.Close() - gw.Write([]byte(responseBody)) + defer func() { _ = gw.Close() }() + _, _ = gw.Write([]byte(responseBody)) return } - w.Write([]byte(responseBody)) + _, _ = w.Write([]byte(responseBody)) }), ) defer server.Close() @@ -3574,66 +3463,62 @@ func (ts *testScraper) readResponse(ctx context.Context, _ *http.Response, w io. func TestScrapeLoop_RespectTimestamps(t *testing.T) { s := teststorage.New(t) - defer s.Close() + t.Cleanup(func() { _ = s.Close() }) - app := s.Appender(context.Background()) - capp := &collectResultAppender{next: app} - sl := newBasicScrapeLoop(t, context.Background(), nil, func(context.Context) storage.Appender { return capp }, 0) + sl, _, appTest := newTestScrapeLoop(t) + appTest.Next = s now := time.Now() - slApp := sl.appender(context.Background()) - _, _, _, err := sl.append(slApp, []byte(`metric_a{a="1",b="1"} 1 0`), "text/plain", now) + slApp := sl.appender() + _, _, _, err := slApp.append([]byte(`metric_a{a="1",b="1"} 1 0`), "text/plain", now) require.NoError(t, err) require.NoError(t, slApp.Commit()) - want := []floatSample{ + want := []sample{ { - metric: labels.FromStrings("__name__", "metric_a", "a", "1", "b", "1"), - t: 0, - f: 1, + L: labels.FromStrings("__name__", "metric_a", "a", "1", "b", "1"), + T: 0, + V: 1, }, } - require.Equal(t, want, capp.resultFloats, "Appended samples not as expected:\n%s", appender) + require.Equal(t, want, appTest.ResultSamples, "Appended samples not as expected:\n%s", appTest) } func TestScrapeLoop_DiscardTimestamps(t *testing.T) { s := teststorage.New(t) - defer s.Close() + t.Cleanup(func() { _ = s.Close() }) - app := s.Appender(context.Background()) - - capp := &collectResultAppender{next: app} - - sl := newBasicScrapeLoop(t, context.Background(), nil, func(context.Context) storage.Appender { return capp }, 0) - sl.honorTimestamps = false + sl, _, appTest := newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.honorTimestamps = false + }) + appTest.Next = s now := time.Now() - slApp := sl.appender(context.Background()) - _, _, _, err := sl.append(slApp, []byte(`metric_a{a="1",b="1"} 1 0`), "text/plain", now) + slApp := sl.appender() + _, _, _, err := slApp.append([]byte(`metric_a{a="1",b="1"} 1 0`), "text/plain", now) require.NoError(t, err) require.NoError(t, slApp.Commit()) - want := []floatSample{ + want := []sample{ { - metric: labels.FromStrings("__name__", "metric_a", "a", "1", "b", "1"), - t: timestamp.FromTime(now), - f: 1, + L: labels.FromStrings("__name__", "metric_a", "a", "1", "b", "1"), + T: timestamp.FromTime(now), + V: 1, }, } - require.Equal(t, want, capp.resultFloats, "Appended samples not as expected:\n%s", appender) + require.Equal(t, want, appTest.ResultSamples, "Appended samples not as expected:\n%s", appTest) } func TestScrapeLoopDiscardDuplicateLabels(t *testing.T) { s := teststorage.New(t) - defer s.Close() + t.Cleanup(func() { _ = s.Close() }) - ctx, cancel := context.WithCancel(context.Background()) - sl := newBasicScrapeLoop(t, ctx, &testScraper{}, s.Appender, 0) - defer cancel() + sl, _, appTest := newTestScrapeLoop(t) + appTest.Next = s // We add a good and a bad metric to check that both are discarded. - slApp := sl.appender(ctx) - _, _, _, err := sl.append(slApp, []byte("test_metric{le=\"500\"} 1\ntest_metric{le=\"600\",le=\"700\"} 1\n"), "text/plain", time.Time{}) + slApp := sl.appender() + _, _, _, err := slApp.append([]byte("test_metric{le=\"500\"} 1\ntest_metric{le=\"600\",le=\"700\"} 1\n"), "text/plain", time.Time{}) require.Error(t, err) require.NoError(t, slApp.Rollback()) // We need to cycle staleness cache maps after a manual rollback. Otherwise they will have old entries in them, @@ -3642,19 +3527,19 @@ func TestScrapeLoopDiscardDuplicateLabels(t *testing.T) { q, err := s.Querier(time.Time{}.UnixNano(), 0) require.NoError(t, err) - series := q.Select(ctx, false, nil, labels.MustNewMatcher(labels.MatchRegexp, "__name__", ".*")) + series := q.Select(sl.ctx, false, nil, labels.MustNewMatcher(labels.MatchRegexp, "__name__", ".*")) require.False(t, series.Next(), "series found in tsdb") require.NoError(t, series.Err()) // We add a good metric to check that it is recorded. - slApp = sl.appender(ctx) - _, _, _, err = sl.append(slApp, []byte("test_metric{le=\"500\"} 1\n"), "text/plain", time.Time{}) + slApp = sl.appender() + _, _, _, err = slApp.append([]byte("test_metric{le=\"500\"} 1\n"), "text/plain", time.Time{}) require.NoError(t, err) require.NoError(t, slApp.Commit()) q, err = s.Querier(time.Time{}.UnixNano(), 0) require.NoError(t, err) - series = q.Select(ctx, false, nil, labels.MustNewMatcher(labels.MatchEqual, "le", "500")) + series = q.Select(sl.ctx, false, nil, labels.MustNewMatcher(labels.MatchEqual, "le", "500")) require.True(t, series.Next(), "series not found in tsdb") require.NoError(t, series.Err()) require.False(t, series.Next(), "more than one series found in tsdb") @@ -3662,29 +3547,27 @@ func TestScrapeLoopDiscardDuplicateLabels(t *testing.T) { func TestScrapeLoopDiscardUnnamedMetrics(t *testing.T) { s := teststorage.New(t) - defer s.Close() + t.Cleanup(func() { _ = s.Close() }) - app := s.Appender(context.Background()) - - ctx, cancel := context.WithCancel(context.Background()) - sl := newBasicScrapeLoop(t, context.Background(), &testScraper{}, func(context.Context) storage.Appender { return app }, 0) - sl.sampleMutator = func(l labels.Labels) labels.Labels { - if l.Has("drop") { - return labels.FromStrings("no", "name") // This label set will trigger an error. + sl, _, appTest := newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.sampleMutator = func(l labels.Labels) labels.Labels { + if l.Has("drop") { + return labels.FromStrings("no", "name") // This label set will trigger an error. + } + return l } - return l - } - defer cancel() + }) + appTest.Next = s - slApp := sl.appender(context.Background()) - _, _, _, err := sl.append(slApp, []byte("nok 1\nnok2{drop=\"drop\"} 1\n"), "text/plain", time.Time{}) + slApp := sl.appender() + _, _, _, err := slApp.append([]byte("nok 1\nnok2{drop=\"drop\"} 1\n"), "text/plain", time.Time{}) require.Error(t, err) require.NoError(t, slApp.Rollback()) require.Equal(t, errNameLabelMandatory, err) q, err := s.Querier(time.Time{}.UnixNano(), 0) require.NoError(t, err) - series := q.Select(ctx, false, nil, labels.MustNewMatcher(labels.MatchRegexp, "__name__", ".*")) + series := q.Select(sl.ctx, false, nil, labels.MustNewMatcher(labels.MatchRegexp, "__name__", ".*")) require.False(t, series.Next(), "series found in tsdb") require.NoError(t, series.Err()) } @@ -3767,7 +3650,7 @@ func TestReuseScrapeCache(t *testing.T) { MetricNameValidationScheme: model.UTF8Validation, MetricNameEscapingScheme: model.AllowUTF8, } - sp, _ = newScrapePool(cfg, app, 0, nil, nil, &Options{}, newTestScrapeMetrics(t)) + sp, _ = newScrapePool(cfg, nil, app, 0, nil, nil, &Options{}, newTestScrapeMetrics(t)) t1 = &Target{ labels: labels.FromStrings("labelNew", "nameNew", "labelNew1", "nameNew1", "labelNew2", "nameNew2"), scrapeConfig: &config.ScrapeConfig{ @@ -3924,7 +3807,7 @@ func TestReuseScrapeCache(t *testing.T) { for i, s := range steps { initCacheAddr := cacheAddr(sp) - sp.reload(s.newConfig) + require.NoError(t, sp.reload(s.newConfig)) for fp, newCacheAddr := range cacheAddr(sp) { if s.keep { require.Equal(t, initCacheAddr[fp], newCacheAddr, "step %d: old cache and new cache are not the same", i) @@ -3933,7 +3816,7 @@ func TestReuseScrapeCache(t *testing.T) { } } initCacheAddr = cacheAddr(sp) - sp.reload(s.newConfig) + require.NoError(t, sp.reload(s.newConfig)) for fp, newCacheAddr := range cacheAddr(sp) { require.Equal(t, initCacheAddr[fp], newCacheAddr, "step %d: reloading the exact config invalidates the cache", i) } @@ -3942,14 +3825,13 @@ func TestReuseScrapeCache(t *testing.T) { func TestScrapeAddFast(t *testing.T) { s := teststorage.New(t) - defer s.Close() + t.Cleanup(func() { _ = s.Close() }) - ctx, cancel := context.WithCancel(context.Background()) - sl := newBasicScrapeLoop(t, ctx, &testScraper{}, s.Appender, 0) - defer cancel() + sl, _, appTest := newTestScrapeLoop(t) + appTest.Next = s - slApp := sl.appender(ctx) - _, _, _, err := sl.append(slApp, []byte("up 1\n"), "text/plain", time.Time{}) + slApp := sl.appender() + _, _, _, err := slApp.append([]byte("up 1\n"), "text/plain", time.Time{}) require.NoError(t, err) require.NoError(t, slApp.Commit()) @@ -3959,8 +3841,8 @@ func TestScrapeAddFast(t *testing.T) { v.ref++ } - slApp = sl.appender(ctx) - _, _, _, err = sl.append(slApp, []byte("up 1\n"), "text/plain", time.Time{}.Add(time.Second)) + slApp = sl.appender() + _, _, _, err = slApp.append([]byte("up 1\n"), "text/plain", time.Time{}.Add(time.Second)) require.NoError(t, err) require.NoError(t, slApp.Commit()) } @@ -3977,7 +3859,7 @@ func TestReuseCacheRace(t *testing.T) { MetricNameEscapingScheme: model.AllowUTF8, } buffers = pool.New(1e3, 100e6, 3, func(sz int) any { return make([]byte, 0, sz) }) - sp, _ = newScrapePool(cfg, app, 0, nil, buffers, &Options{}, newTestScrapeMetrics(t)) + sp, _ = newScrapePool(cfg, nil, app, 0, nil, buffers, &Options{}, newTestScrapeMetrics(t)) t1 = &Target{ labels: labels.FromStrings("labelNew", "nameNew"), scrapeConfig: &config.ScrapeConfig{}, @@ -3991,7 +3873,7 @@ func TestReuseCacheRace(t *testing.T) { if time.Since(start) > 5*time.Second { break } - sp.reload(&config.ScrapeConfig{ + require.NoError(t, sp.reload(&config.ScrapeConfig{ JobName: "Prometheus", ScrapeTimeout: model.Duration(1 * time.Millisecond), ScrapeInterval: model.Duration(1 * time.Millisecond), @@ -3999,39 +3881,42 @@ func TestReuseCacheRace(t *testing.T) { SampleLimit: i, MetricNameValidationScheme: model.UTF8Validation, MetricNameEscapingScheme: model.AllowUTF8, - }) + })) } } func TestCheckAddError(t *testing.T) { var appErrs appendErrors sl := scrapeLoop{l: promslog.NewNopLogger(), metrics: newTestScrapeMetrics(t)} - sl.checkAddError(nil, storage.ErrOutOfOrderSample, nil, nil, &appErrs) + // TODO: Check err etc + _, _ = sl.checkAddError(nil, nil, storage.ErrOutOfOrderSample, nil, nil, &appErrs) require.Equal(t, 1, appErrs.numOutOfOrder) + + // TODO(bwplotka): Test partial error check and other cases } func TestScrapeReportSingleAppender(t *testing.T) { t.Parallel() s := teststorage.New(t) - defer s.Close() + t.Cleanup(func() { _ = s.Close() }) - var ( - signal = make(chan struct{}, 1) - scraper = &testScraper{} - ) + signal := make(chan struct{}, 1) - ctx, cancel := context.WithCancel(context.Background()) - // Since we're writing samples directly below we need to provide a protocol fallback. - sl := newBasicScrapeLoopWithFallback(t, ctx, scraper, s.Appender, 10*time.Millisecond, "text/plain") + ctx, cancel := context.WithCancel(t.Context()) + sl, scraper, _ := newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.ctx = ctx + sl.appendableV2 = s + // Since we're writing samples directly below we need to provide a protocol fallback. + sl.fallbackScrapeProtocol = "text/plain" + }) numScrapes := 0 - scraper.scrapeFunc = func(_ context.Context, w io.Writer) error { numScrapes++ if numScrapes%4 == 0 { return errors.New("scrape failed") } - w.Write([]byte("metric_a 44\nmetric_b 44\nmetric_c 44\nmetric_d 44\n")) + _, _ = w.Write([]byte("metric_a 44\nmetric_b 44\nmetric_c 44\nmetric_d 44\n")) return nil } @@ -4055,7 +3940,7 @@ func TestScrapeReportSingleAppender(t *testing.T) { } require.Equal(t, 0, c%9, "Appended samples not as expected: %d", c) - q.Close() + require.NoError(t, q.Close()) } cancel() @@ -4068,7 +3953,7 @@ func TestScrapeReportSingleAppender(t *testing.T) { func TestScrapeReportLimit(t *testing.T) { s := teststorage.New(t) - defer s.Close() + t.Cleanup(func() { _ = s.Close() }) cfg := &config.ScrapeConfig{ JobName: "test", @@ -4083,7 +3968,7 @@ func TestScrapeReportLimit(t *testing.T) { ts, scrapedTwice := newScrapableServer("metric_a 44\nmetric_b 44\nmetric_c 44\nmetric_d 44\n") defer ts.Close() - sp, err := newScrapePool(cfg, s, 0, nil, nil, &Options{}, newTestScrapeMetrics(t)) + sp, err := newScrapePool(cfg, nil, s, 0, nil, nil, &Options{}, newTestScrapeMetrics(t)) require.NoError(t, err) defer sp.stop() @@ -4106,7 +3991,7 @@ func TestScrapeReportLimit(t *testing.T) { ctx := t.Context() q, err := s.Querier(time.Time{}.UnixNano(), time.Now().UnixNano()) require.NoError(t, err) - defer q.Close() + t.Cleanup(func() { _ = q.Close() }) series := q.Select(ctx, false, nil, labels.MustNewMatcher(labels.MatchRegexp, "__name__", "up")) var found bool @@ -4124,7 +4009,7 @@ func TestScrapeReportLimit(t *testing.T) { func TestScrapeUTF8(t *testing.T) { s := teststorage.New(t) - defer s.Close() + t.Cleanup(func() { _ = s.Close() }) cfg := &config.ScrapeConfig{ JobName: "test", @@ -4137,7 +4022,7 @@ func TestScrapeUTF8(t *testing.T) { ts, scrapedTwice := newScrapableServer("{\"with.dots\"} 42\n") defer ts.Close() - sp, err := newScrapePool(cfg, s, 0, nil, nil, &Options{}, newTestScrapeMetrics(t)) + sp, err := newScrapePool(cfg, nil, s, 0, nil, nil, &Options{}, newTestScrapeMetrics(t)) require.NoError(t, err) defer sp.stop() @@ -4160,7 +4045,7 @@ func TestScrapeUTF8(t *testing.T) { ctx := t.Context() q, err := s.Querier(time.Time{}.UnixNano(), time.Now().UnixNano()) require.NoError(t, err) - defer q.Close() + t.Cleanup(func() { _ = q.Close() }) series := q.Select(ctx, false, nil, labels.MustNewMatcher(labels.MatchRegexp, "__name__", "with.dots")) require.True(t, series.Next(), "series not found in tsdb") @@ -4232,23 +4117,22 @@ func TestScrapeLoopLabelLimit(t *testing.T) { } for _, test := range tests { - app := &collectResultAppender{} - discoveryLabels := &Target{ labels: labels.FromStrings(test.discoveryLabels...), } - sl := newBasicScrapeLoop(t, context.Background(), nil, func(context.Context) storage.Appender { return app }, 0) - sl.sampleMutator = func(l labels.Labels) labels.Labels { - return mutateSampleLabels(l, discoveryLabels, false, nil) - } - sl.reportSampleMutator = func(l labels.Labels) labels.Labels { - return mutateReportSampleLabels(l, discoveryLabels) - } - sl.labelLimits = &test.labelLimits + sl, _, _ := newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.sampleMutator = func(l labels.Labels) labels.Labels { + return mutateSampleLabels(l, discoveryLabels, false, nil) + } + sl.reportSampleMutator = func(l labels.Labels) labels.Labels { + return mutateReportSampleLabels(l, discoveryLabels) + } + sl.labelLimits = &test.labelLimits + }) - slApp := sl.appender(context.Background()) - _, _, _, err := sl.append(slApp, []byte(test.scrapeLabels), "text/plain", time.Now()) + slApp := sl.appender() + _, _, _, err := slApp.append([]byte(test.scrapeLabels), "text/plain", time.Now()) t.Logf("Test:%s", test.title) if test.expectErr { @@ -4263,7 +4147,7 @@ func TestScrapeLoopLabelLimit(t *testing.T) { func TestTargetScrapeIntervalAndTimeoutRelabel(t *testing.T) { interval, _ := model.ParseDuration("2s") timeout, _ := model.ParseDuration("500ms") - config := &config.ScrapeConfig{ + cfg := &config.ScrapeConfig{ ScrapeInterval: interval, ScrapeTimeout: timeout, MetricNameValidationScheme: model.UTF8Validation, @@ -4287,7 +4171,7 @@ func TestTargetScrapeIntervalAndTimeoutRelabel(t *testing.T) { }, }, } - sp, _ := newScrapePool(config, &nopAppendable{}, 0, nil, nil, &Options{}, newTestScrapeMetrics(t)) + sp, _ := newScrapePool(cfg, nil, &nopAppendable{}, 0, nil, nil, &Options{}, newTestScrapeMetrics(t)) tgts := []*targetgroup.Group{ { Targets: []model.LabelSet{{model.AddressLabel: "127.0.0.1:9090"}}, @@ -4303,10 +4187,10 @@ func TestTargetScrapeIntervalAndTimeoutRelabel(t *testing.T) { // Testing whether we can remove trailing .0 from histogram 'le' and summary 'quantile' labels. func TestLeQuantileReLabel(t *testing.T) { - simpleStorage := teststorage.New(t) - defer simpleStorage.Close() + s := teststorage.New(t) + t.Cleanup(func() { _ = s.Close() }) - config := &config.ScrapeConfig{ + cfg := &config.ScrapeConfig{ JobName: "test", MetricRelabelConfigs: []*relabel.Config{ { @@ -4373,7 +4257,7 @@ test_summary_count 199 ts, scrapedTwice := newScrapableServer(metricsText) defer ts.Close() - sp, err := newScrapePool(config, simpleStorage, 0, nil, nil, &Options{}, newTestScrapeMetrics(t)) + sp, err := newScrapePool(cfg, nil, s, 0, nil, nil, &Options{}, newTestScrapeMetrics(t)) require.NoError(t, err) defer sp.stop() @@ -4393,9 +4277,9 @@ test_summary_count 199 } ctx := t.Context() - q, err := simpleStorage.Querier(time.Time{}.UnixNano(), time.Now().UnixNano()) + q, err := s.Querier(time.Time{}.UnixNano(), time.Now().UnixNano()) require.NoError(t, err) - defer q.Close() + t.Cleanup(func() { _ = q.Close() }) checkValues := func(labelName string, expectedValues []string, series storage.SeriesSet) { foundLeValues := map[string]bool{} @@ -4441,12 +4325,12 @@ func TestConvertClassicHistogramsToNHCB(t *testing.T) { } b := &bytes.Buffer{} if withMetadata { - template.Must(template.New("").Parse(` + require.NoError(t, template.Must(template.New("").Parse(` # HELP {{.name}} This is a histogram with default buckets # TYPE {{.name}} histogram -`)).Execute(b, data) +`)).Execute(b, data)) } - template.Must(template.New("").Parse(` + require.NoError(t, template.Must(template.New("").Parse(` {{.name}}_bucket{address="0.0.0.0",port="5001",le="0.005"} 0 {{.name}}_bucket{address="0.0.0.0",port="5001",le="0.01"} 0 {{.name}}_bucket{address="0.0.0.0",port="5001",le="0.025"} 0 @@ -4461,7 +4345,7 @@ func TestConvertClassicHistogramsToNHCB(t *testing.T) { {{.name}}_bucket{address="0.0.0.0",port="5001",le="+Inf"} 1 {{.name}}_sum{address="0.0.0.0",port="5001"} 10 {{.name}}_count{address="0.0.0.0",port="5001"} 1 -`)).Execute(b, data) +`)).Execute(b, data)) return b.String() } genTestCounterProto := func(name string, value int) string { @@ -4831,14 +4715,17 @@ metric: < t.Run(fmt.Sprintf("%s with %s", name, metricsTextName), func(t *testing.T) { t.Parallel() - simpleStorage := teststorage.New(t) - defer simpleStorage.Close() + s := teststorage.New(t) + t.Cleanup(func() { _ = s.Close() }) - sl := newBasicScrapeLoop(t, context.Background(), nil, func(ctx context.Context) storage.Appender { return simpleStorage.Appender(ctx) }, 0) - sl.alwaysScrapeClassicHist = tc.alwaysScrapeClassicHistograms - sl.convertClassicHistToNHCB = tc.convertClassicHistToNHCB - sl.enableNativeHistogramScraping = true - app := simpleStorage.Appender(context.Background()) + sl, _, _ := newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.appendableV2 = s + sl.alwaysScrapeClassicHist = tc.alwaysScrapeClassicHistograms + sl.convertClassicHistToNHCB = tc.convertClassicHistToNHCB + sl.enableNativeHistogramScraping = true + }) + + slApp := sl.appender() var content []byte contentType := metricsText.contentType @@ -4862,13 +4749,14 @@ metric: < default: t.Error("unexpected content type") } - sl.append(app, content, contentType, time.Now()) - require.NoError(t, app.Commit()) + _, _, _, err := slApp.append(content, contentType, time.Now()) + require.NoError(t, err) + require.NoError(t, slApp.Commit()) ctx := t.Context() - q, err := simpleStorage.Querier(time.Time{}.UnixNano(), time.Now().UnixNano()) + q, err := s.Querier(time.Time{}.UnixNano(), time.Now().UnixNano()) require.NoError(t, err) - defer q.Close() + t.Cleanup(func() { _ = q.Close() }) var series storage.SeriesSet @@ -4910,10 +4798,10 @@ metric: < } func TestTypeUnitReLabel(t *testing.T) { - simpleStorage := teststorage.New(t) - defer simpleStorage.Close() + s := teststorage.New(t) + t.Cleanup(func() { _ = s.Close() }) - config := &config.ScrapeConfig{ + cfg := &config.ScrapeConfig{ JobName: "test", MetricRelabelConfigs: []*relabel.Config{ { @@ -4958,7 +4846,7 @@ disk_usage_bytes 456 ts, scrapedTwice := newScrapableServer(metricsText) defer ts.Close() - sp, err := newScrapePool(config, simpleStorage, 0, nil, nil, &Options{}, newTestScrapeMetrics(t)) + sp, err := newScrapePool(cfg, nil, s, 0, nil, nil, &Options{}, newTestScrapeMetrics(t)) require.NoError(t, err) defer sp.stop() @@ -4978,9 +4866,9 @@ disk_usage_bytes 456 } ctx := t.Context() - q, err := simpleStorage.Querier(time.Time{}.UnixNano(), time.Now().UnixNano()) + q, err := s.Querier(time.Time{}.UnixNano(), time.Now().UnixNano()) require.NoError(t, err) - defer q.Close() + t.Cleanup(func() { _ = q.Close() }) series := q.Select(ctx, false, nil, labels.MustNewMatcher(labels.MatchRegexp, "__name__", ".*_total$")) for series.Next() { @@ -4996,26 +4884,24 @@ disk_usage_bytes 456 } func TestScrapeLoopRunCreatesStaleMarkersOnFailedScrapeForTimestampedMetrics(t *testing.T) { - appender := &collectResultAppender{} - var ( - signal = make(chan struct{}, 1) - scraper = &testScraper{} - app = func(context.Context) storage.Appender { return appender } - ) + signal := make(chan struct{}, 1) + + ctx, cancel := context.WithCancel(t.Context()) + sl, scraper, appTest := newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.ctx = ctx + // Since we're writing samples directly below we need to provide a protocol fallback. + sl.fallbackScrapeProtocol = "text/plain" + sl.trackTimestampsStaleness = true + }) - ctx, cancel := context.WithCancel(context.Background()) - // Since we're writing samples directly below we need to provide a protocol fallback. - sl := newBasicScrapeLoopWithFallback(t, ctx, scraper, app, 10*time.Millisecond, "text/plain") - sl.trackTimestampsStaleness = true // Succeed once, several failures, then stop. numScrapes := 0 - scraper.scrapeFunc = func(_ context.Context, w io.Writer) error { numScrapes++ switch numScrapes { case 1: - fmt.Fprintf(w, "metric_a 42 %d\n", time.Now().UnixNano()/int64(time.Millisecond)) + _, _ = fmt.Fprintf(w, "metric_a 42 %d\n", time.Now().UnixNano()/int64(time.Millisecond)) return nil case 5: cancel() @@ -5035,15 +4921,15 @@ func TestScrapeLoopRunCreatesStaleMarkersOnFailedScrapeForTimestampedMetrics(t * } // 1 successfully scraped sample, 1 stale marker after first fail, 5 report samples for // each scrape successful or not. - require.Len(t, appender.resultFloats, 27, "Appended samples not as expected:\n%s", appender) - require.Equal(t, 42.0, appender.resultFloats[0].f, "Appended first sample not as expected") - require.True(t, value.IsStaleNaN(appender.resultFloats[6].f), - "Appended second sample not as expected. Wanted: stale NaN Got: %x", math.Float64bits(appender.resultFloats[6].f)) + require.Len(t, appTest.ResultSamples, 27, "Appended samples not as expected:\n%s", appTest) + require.Equal(t, 42.0, appTest.ResultSamples[0].V, "Appended first sample not as expected") + require.True(t, value.IsStaleNaN(appTest.ResultSamples[6].V), + "Appended second sample not as expected. Wanted: stale NaN Got: %x", math.Float64bits(appTest.ResultSamples[6].V)) } func TestScrapeLoopCompression(t *testing.T) { - simpleStorage := teststorage.New(t) - defer simpleStorage.Close() + s := teststorage.New(t) + t.Cleanup(func() { _ = s.Close() }) metricsText := makeTestGauges(10) @@ -5065,12 +4951,12 @@ func TestScrapeLoopCompression(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { require.Equal(t, tc.acceptEncoding, r.Header.Get("Accept-Encoding"), "invalid value of the Accept-Encoding header") - fmt.Fprint(w, string(metricsText)) + _, _ = fmt.Fprint(w, string(metricsText)) close(scraped) })) defer ts.Close() - config := &config.ScrapeConfig{ + cfg := &config.ScrapeConfig{ JobName: "test", SampleLimit: 100, Scheme: "http", @@ -5081,7 +4967,7 @@ func TestScrapeLoopCompression(t *testing.T) { MetricNameEscapingScheme: model.AllowUTF8, } - sp, err := newScrapePool(config, simpleStorage, 0, nil, nil, &Options{}, newTestScrapeMetrics(t)) + sp, err := newScrapePool(cfg, nil, s, 0, nil, nil, &Options{}, newTestScrapeMetrics(t)) require.NoError(t, err) defer sp.stop() @@ -5191,11 +5077,11 @@ func BenchmarkTargetScraperGzip(b *testing.B) { gw := gzip.NewWriter(&buf) for j := 0; j < scenarios[i].metricsCount; j++ { name = fmt.Sprintf("go_memstats_alloc_bytes_total_%d", j) - fmt.Fprintf(gw, "# HELP %s Total number of bytes allocated, even if freed.\n", name) - fmt.Fprintf(gw, "# TYPE %s counter\n", name) - fmt.Fprintf(gw, "%s %d\n", name, i*j) + _, _ = fmt.Fprintf(gw, "# HELP %s Total number of bytes allocated, even if freed.\n", name) + _, _ = fmt.Fprintf(gw, "# TYPE %s counter\n", name) + _, _ = fmt.Fprintf(gw, "%s %d\n", name, i*j) } - gw.Close() + require.NoError(b, gw.Close()) scenarios[i].body = buf.Bytes() } @@ -5204,7 +5090,7 @@ func BenchmarkTargetScraperGzip(b *testing.B) { w.Header().Set("Content-Encoding", "gzip") for _, scenario := range scenarios { if strconv.Itoa(scenario.metricsCount) == r.URL.Query()["count"][0] { - w.Write(scenario.body) + _, _ = w.Write(scenario.body) return } } @@ -5253,10 +5139,10 @@ func BenchmarkTargetScraperGzip(b *testing.B) { // When a scrape contains multiple instances for the same time series we should increment // prometheus_target_scrapes_sample_duplicate_timestamp_total metric. func TestScrapeLoopSeriesAddedDuplicates(t *testing.T) { - ctx, sl := simpleTestScrapeLoop(t) + sl, _, _ := newTestScrapeLoop(t) - slApp := sl.appender(ctx) - total, added, seriesAdded, err := sl.append(slApp, []byte("test_metric 1\ntest_metric 2\ntest_metric 3\n"), "text/plain", time.Time{}) + slApp := sl.appender() + total, added, seriesAdded, err := slApp.append([]byte("test_metric 1\ntest_metric 2\ntest_metric 3\n"), "text/plain", time.Time{}) require.NoError(t, err) require.NoError(t, slApp.Commit()) require.Equal(t, 3, total) @@ -5264,8 +5150,8 @@ func TestScrapeLoopSeriesAddedDuplicates(t *testing.T) { require.Equal(t, 1, seriesAdded) require.Equal(t, 2.0, prom_testutil.ToFloat64(sl.metrics.targetScrapeSampleDuplicate)) - slApp = sl.appender(ctx) - total, added, seriesAdded, err = sl.append(slApp, []byte("test_metric 1\ntest_metric 1\ntest_metric 1\n"), "text/plain", time.Time{}) + slApp = sl.appender() + total, added, seriesAdded, err = slApp.append([]byte("test_metric 1\ntest_metric 1\ntest_metric 1\n"), "text/plain", time.Time{}) require.NoError(t, err) require.NoError(t, slApp.Commit()) require.Equal(t, 3, total) @@ -5274,8 +5160,8 @@ func TestScrapeLoopSeriesAddedDuplicates(t *testing.T) { require.Equal(t, 4.0, prom_testutil.ToFloat64(sl.metrics.targetScrapeSampleDuplicate)) // When different timestamps are supplied, multiple samples are accepted. - slApp = sl.appender(ctx) - total, added, seriesAdded, err = sl.append(slApp, []byte("test_metric 1 1001\ntest_metric 1 1002\ntest_metric 1 1003\n"), "text/plain", time.Time{}) + slApp = sl.appender() + total, added, seriesAdded, err = slApp.append([]byte("test_metric 1 1001\ntest_metric 1 1002\ntest_metric 1 1003\n"), "text/plain", time.Time{}) require.NoError(t, err) require.NoError(t, slApp.Commit()) require.Equal(t, 3, total) @@ -5325,7 +5211,7 @@ func testNativeHistogramMaxSchemaSet(t *testing.T, minBucketFactor string, expec }, ) registry := prometheus.NewRegistry() - registry.Register(nativeHistogram) + require.NoError(t, registry.Register(nativeHistogram)) nativeHistogram.Observe(1.0) nativeHistogram.Observe(1.0) nativeHistogram.Observe(1.0) @@ -5342,7 +5228,7 @@ func testNativeHistogramMaxSchemaSet(t *testing.T, minBucketFactor string, expec // Create a HTTP server to serve /metrics via ProtoBuf metricsServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", `application/vnd.google.protobuf; proto=io.prometheus.client.MetricFamily; encoding=delimited`) - w.Write(buffer) + _, _ = w.Write(buffer) })) defer metricsServer.Close() @@ -5361,18 +5247,17 @@ scrape_configs: `, minBucketFactor, strings.ReplaceAll(metricsServer.URL, "http://", "")) s := teststorage.New(t) - defer s.Close() + t.Cleanup(func() { _ = s.Close() }) reg := prometheus.NewRegistry() - mng, err := NewManager(&Options{DiscoveryReloadInterval: model.Duration(10 * time.Millisecond)}, nil, nil, s, reg) + mng, err := NewManagerWithAppendableV2(&Options{DiscoveryReloadInterval: model.Duration(10 * time.Millisecond)}, nil, nil, s, reg) require.NoError(t, err) cfg, err := config.Load(configStr, promslog.NewNopLogger()) require.NoError(t, err) - mng.ApplyConfig(cfg) + require.NoError(t, mng.ApplyConfig(cfg)) tsets := make(chan map[string][]*targetgroup.Group) go func() { - err = mng.Run(tsets) - require.NoError(t, err) + require.NoError(t, mng.Run(tsets)) }() defer mng.Stop() @@ -5447,7 +5332,7 @@ func TestTargetScrapeConfigWithLabels(t *testing.T) { require.Equal(t, expectedPath, r.URL.Path) w.Header().Set("Content-Type", `text/plain; version=0.0.4`) - w.Write([]byte("metric_a 1\nmetric_b 2\n")) + _, _ = w.Write([]byte("metric_a 1\nmetric_b 2\n")) }), ) t.Cleanup(server.Close) @@ -5467,7 +5352,7 @@ func TestTargetScrapeConfigWithLabels(t *testing.T) { } } - sp, err := newScrapePool(cfg, &nopAppendable{}, 0, nil, nil, &Options{}, newTestScrapeMetrics(t)) + sp, err := newScrapePool(cfg, nil, &nopAppendable{}, 0, nil, nil, &Options{}, newTestScrapeMetrics(t)) require.NoError(t, err) t.Cleanup(sp.stop) @@ -5595,7 +5480,7 @@ func newScrapableServer(scrapeText string) (s *httptest.Server, scrapedTwice cha scrapedTwice = make(chan bool) return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - fmt.Fprint(w, scrapeText) + _, _ = fmt.Fprint(w, scrapeText) scrapes++ if scrapes == 2 { close(scrapedTwice) @@ -5607,7 +5492,7 @@ func newScrapableServer(scrapeText string) (s *httptest.Server, scrapedTwice cha func TestScrapePoolScrapeAfterReload(t *testing.T) { h := httptest.NewServer(http.HandlerFunc( func(w http.ResponseWriter, _ *http.Request) { - w.Write([]byte{0x42, 0x42}) + _, _ = w.Write([]byte{0x42, 0x42}) }, )) t.Cleanup(h.Close) @@ -5630,7 +5515,7 @@ func TestScrapePoolScrapeAfterReload(t *testing.T) { }, } - p, err := newScrapePool(cfg, &nopAppendable{}, 0, nil, nil, &Options{}, newTestScrapeMetrics(t)) + p, err := newScrapePool(cfg, nil, &nopAppendable{}, 0, nil, nil, &Options{}, newTestScrapeMetrics(t)) require.NoError(t, err) t.Cleanup(p.stop) @@ -5657,46 +5542,42 @@ func TestScrapeAppendWithParseError(t *testing.T) { # EOF` ) - sl := newBasicScrapeLoop(t, context.Background(), nil, nil, 0) - sl.cache = newScrapeCache(sl.metrics) + sl, _, appTest := newTestScrapeLoop(t) + slApp := sl.appender() now := time.Now() - capp := &collectResultAppender{next: nopAppender{}} - _, _, _, err := sl.append(capp, []byte(scrape1), "application/openmetrics-text", now) + _, _, _, err := slApp.append([]byte(scrape1), "application/openmetrics-text", now) require.Error(t, err) - _, _, _, err = sl.append(capp, nil, "application/openmetrics-text", now) + _, _, _, err = slApp.append(nil, "application/openmetrics-text", now) require.NoError(t, err) - require.Empty(t, capp.resultFloats) + require.Empty(t, appTest.ResultSamples) - capp = &collectResultAppender{next: nopAppender{}} - _, _, _, err = sl.append(capp, []byte(scrape2), "application/openmetrics-text", now.Add(15*time.Second)) + _, _, _, err = slApp.append([]byte(scrape2), "application/openmetrics-text", now.Add(15*time.Second)) require.NoError(t, err) - require.NoError(t, capp.Commit()) + require.NoError(t, slApp.Commit()) - want := []floatSample{ + want := []sample{ { - metric: labels.FromStrings(model.MetricNameLabel, "metric_a"), - t: timestamp.FromTime(now.Add(15 * time.Second)), - f: 11, + L: labels.FromStrings(model.MetricNameLabel, "metric_a"), + T: timestamp.FromTime(now.Add(15 * time.Second)), + V: 11, }, } - requireEqual(t, want, capp.resultFloats, "Appended samples not as expected:\n%s", capp) + requireEqual(t, want, appTest.ResultSamples, "Appended samples not as expected:\n%s", appTest) } // This test covers a case where there's a target with sample_limit set and the some of exporter samples // changes between scrapes. func TestScrapeLoopAppendSampleLimitWithDisappearingSeries(t *testing.T) { const sampleLimit = 4 - resApp := &collectResultAppender{} - sl := newBasicScrapeLoop(t, context.Background(), nil, func(_ context.Context) storage.Appender { - return resApp - }, 0) - sl.sampleLimit = sampleLimit + + sl, _, appTest := newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.sampleLimit = sampleLimit + }) now := time.Now() - slApp := sl.appender(context.Background()) - samplesScraped, samplesAfterRelabel, createdSeries, err := sl.append( - slApp, + slApp := sl.appender() + samplesScraped, samplesAfterRelabel, createdSeries, err := slApp.append( // Start with 3 samples, all accepted. []byte("metric_a 1\nmetric_b 1\nmetric_c 1\n"), "text/plain", @@ -5707,29 +5588,28 @@ func TestScrapeLoopAppendSampleLimitWithDisappearingSeries(t *testing.T) { require.Equal(t, 3, samplesScraped) // All on scrape. require.Equal(t, 3, samplesAfterRelabel) // This is series after relabeling. require.Equal(t, 3, createdSeries) // Newly added to TSDB. - want := []floatSample{ + want := []sample{ { - metric: labels.FromStrings(model.MetricNameLabel, "metric_a"), - t: timestamp.FromTime(now), - f: 1, + L: labels.FromStrings(model.MetricNameLabel, "metric_a"), + T: timestamp.FromTime(now), + V: 1, }, { - metric: labels.FromStrings(model.MetricNameLabel, "metric_b"), - t: timestamp.FromTime(now), - f: 1, + L: labels.FromStrings(model.MetricNameLabel, "metric_b"), + T: timestamp.FromTime(now), + V: 1, }, { - metric: labels.FromStrings(model.MetricNameLabel, "metric_c"), - t: timestamp.FromTime(now), - f: 1, + L: labels.FromStrings(model.MetricNameLabel, "metric_c"), + T: timestamp.FromTime(now), + V: 1, }, } - requireEqual(t, want, resApp.resultFloats, "Appended samples not as expected:\n%s", slApp) + requireEqual(t, want, appTest.ResultSamples, "Appended samples not as expected:\n%s", slApp) now = now.Add(time.Minute) - slApp = sl.appender(context.Background()) - samplesScraped, samplesAfterRelabel, createdSeries, err = sl.append( - slApp, + slApp = sl.appender() + samplesScraped, samplesAfterRelabel, createdSeries, err = slApp.append( // Start exporting 3 more samples, so we're over the limit now. []byte("metric_a 1\nmetric_b 1\nmetric_c 1\nmetric_d 1\nmetric_e 1\nmetric_f 1\n"), "text/plain", @@ -5740,13 +5620,12 @@ func TestScrapeLoopAppendSampleLimitWithDisappearingSeries(t *testing.T) { require.Equal(t, 6, samplesScraped) require.Equal(t, 6, samplesAfterRelabel) require.Equal(t, 1, createdSeries) // We've added one series before hitting the limit. - requireEqual(t, want, resApp.resultFloats, "Appended samples not as expected:\n%s", slApp) + requireEqual(t, want, appTest.ResultSamples, "Appended samples not as expected:\n%s", slApp) sl.cache.iterDone(false) now = now.Add(time.Minute) - slApp = sl.appender(context.Background()) - samplesScraped, samplesAfterRelabel, createdSeries, err = sl.append( - slApp, + slApp = sl.appender() + samplesScraped, samplesAfterRelabel, createdSeries, err = slApp.append( // Remove all samples except first 2. []byte("metric_a 1\nmetric_b 1\n"), "text/plain", @@ -5763,45 +5642,43 @@ func TestScrapeLoopAppendSampleLimitWithDisappearingSeries(t *testing.T) { // - Append with stale marker for metric_d - this series was added during second scrape before we hit the sample_limit. // We should NOT see: // - Appends with stale markers for metric_e & metric_f - both over the limit during second scrape and so they never made it into TSDB. - want = append(want, []floatSample{ + want = append(want, []sample{ { - metric: labels.FromStrings(model.MetricNameLabel, "metric_a"), - t: timestamp.FromTime(now), - f: 1, + L: labels.FromStrings(model.MetricNameLabel, "metric_a"), + T: timestamp.FromTime(now), + V: 1, }, { - metric: labels.FromStrings(model.MetricNameLabel, "metric_b"), - t: timestamp.FromTime(now), - f: 1, + L: labels.FromStrings(model.MetricNameLabel, "metric_b"), + T: timestamp.FromTime(now), + V: 1, }, { - metric: labels.FromStrings(model.MetricNameLabel, "metric_c"), - t: timestamp.FromTime(now), - f: math.Float64frombits(value.StaleNaN), + L: labels.FromStrings(model.MetricNameLabel, "metric_c"), + T: timestamp.FromTime(now), + V: math.Float64frombits(value.StaleNaN), }, { - metric: labels.FromStrings(model.MetricNameLabel, "metric_d"), - t: timestamp.FromTime(now), - f: math.Float64frombits(value.StaleNaN), + L: labels.FromStrings(model.MetricNameLabel, "metric_d"), + T: timestamp.FromTime(now), + V: math.Float64frombits(value.StaleNaN), }, }...) - requireEqual(t, want, resApp.resultFloats, "Appended samples not as expected:\n%s", slApp) + requireEqual(t, want, appTest.ResultSamples, "Appended samples not as expected:\n%s", slApp) } // This test covers a case where there's a target with sample_limit set and each scrape sees a completely // different set of samples. func TestScrapeLoopAppendSampleLimitReplaceAllSamples(t *testing.T) { const sampleLimit = 4 - resApp := &collectResultAppender{} - sl := newBasicScrapeLoop(t, context.Background(), nil, func(_ context.Context) storage.Appender { - return resApp - }, 0) - sl.sampleLimit = sampleLimit + + sl, _, appTest := newTestScrapeLoop(t, func(sl *scrapeLoop) { + sl.sampleLimit = sampleLimit + }) now := time.Now() - slApp := sl.appender(context.Background()) - samplesScraped, samplesAfterRelabel, createdSeries, err := sl.append( - slApp, + slApp := sl.appender() + samplesScraped, samplesAfterRelabel, createdSeries, err := slApp.append( // Start with 4 samples, all accepted. []byte("metric_a 1\nmetric_b 1\nmetric_c 1\nmetric_d 1\n"), "text/plain", @@ -5812,34 +5689,33 @@ func TestScrapeLoopAppendSampleLimitReplaceAllSamples(t *testing.T) { require.Equal(t, 4, samplesScraped) // All on scrape. require.Equal(t, 4, samplesAfterRelabel) // This is series after relabeling. require.Equal(t, 4, createdSeries) // Newly added to TSDB. - want := []floatSample{ + want := []sample{ { - metric: labels.FromStrings(model.MetricNameLabel, "metric_a"), - t: timestamp.FromTime(now), - f: 1, + L: labels.FromStrings(model.MetricNameLabel, "metric_a"), + T: timestamp.FromTime(now), + V: 1, }, { - metric: labels.FromStrings(model.MetricNameLabel, "metric_b"), - t: timestamp.FromTime(now), - f: 1, + L: labels.FromStrings(model.MetricNameLabel, "metric_b"), + T: timestamp.FromTime(now), + V: 1, }, { - metric: labels.FromStrings(model.MetricNameLabel, "metric_c"), - t: timestamp.FromTime(now), - f: 1, + L: labels.FromStrings(model.MetricNameLabel, "metric_c"), + T: timestamp.FromTime(now), + V: 1, }, { - metric: labels.FromStrings(model.MetricNameLabel, "metric_d"), - t: timestamp.FromTime(now), - f: 1, + L: labels.FromStrings(model.MetricNameLabel, "metric_d"), + T: timestamp.FromTime(now), + V: 1, }, } - requireEqual(t, want, resApp.resultFloats, "Appended samples not as expected:\n%s", slApp) + requireEqual(t, want, appTest.ResultSamples, "Appended samples not as expected:\n%s", slApp) now = now.Add(time.Minute) - slApp = sl.appender(context.Background()) - samplesScraped, samplesAfterRelabel, createdSeries, err = sl.append( - slApp, + slApp = sl.appender() + samplesScraped, samplesAfterRelabel, createdSeries, err = slApp.append( // Replace all samples with new time series. []byte("metric_e 1\nmetric_f 1\nmetric_g 1\nmetric_h 1\n"), "text/plain", @@ -5854,60 +5730,55 @@ func TestScrapeLoopAppendSampleLimitReplaceAllSamples(t *testing.T) { // We expect to see: // - 4 appends for new samples. // - 4 appends with staleness markers for old samples. - want = append(want, []floatSample{ + want = append(want, []sample{ { - metric: labels.FromStrings(model.MetricNameLabel, "metric_e"), - t: timestamp.FromTime(now), - f: 1, + L: labels.FromStrings(model.MetricNameLabel, "metric_e"), + T: timestamp.FromTime(now), + V: 1, }, { - metric: labels.FromStrings(model.MetricNameLabel, "metric_f"), - t: timestamp.FromTime(now), - f: 1, + L: labels.FromStrings(model.MetricNameLabel, "metric_f"), + T: timestamp.FromTime(now), + V: 1, }, { - metric: labels.FromStrings(model.MetricNameLabel, "metric_g"), - t: timestamp.FromTime(now), - f: 1, + L: labels.FromStrings(model.MetricNameLabel, "metric_g"), + T: timestamp.FromTime(now), + V: 1, }, { - metric: labels.FromStrings(model.MetricNameLabel, "metric_h"), - t: timestamp.FromTime(now), - f: 1, + L: labels.FromStrings(model.MetricNameLabel, "metric_h"), + T: timestamp.FromTime(now), + V: 1, }, { - metric: labels.FromStrings(model.MetricNameLabel, "metric_a"), - t: timestamp.FromTime(now), - f: math.Float64frombits(value.StaleNaN), + L: labels.FromStrings(model.MetricNameLabel, "metric_a"), + T: timestamp.FromTime(now), + V: math.Float64frombits(value.StaleNaN), }, { - metric: labels.FromStrings(model.MetricNameLabel, "metric_b"), - t: timestamp.FromTime(now), - f: math.Float64frombits(value.StaleNaN), + L: labels.FromStrings(model.MetricNameLabel, "metric_b"), + T: timestamp.FromTime(now), + V: math.Float64frombits(value.StaleNaN), }, { - metric: labels.FromStrings(model.MetricNameLabel, "metric_c"), - t: timestamp.FromTime(now), - f: math.Float64frombits(value.StaleNaN), + L: labels.FromStrings(model.MetricNameLabel, "metric_c"), + T: timestamp.FromTime(now), + V: math.Float64frombits(value.StaleNaN), }, { - metric: labels.FromStrings(model.MetricNameLabel, "metric_d"), - t: timestamp.FromTime(now), - f: math.Float64frombits(value.StaleNaN), + L: labels.FromStrings(model.MetricNameLabel, "metric_d"), + T: timestamp.FromTime(now), + V: math.Float64frombits(value.StaleNaN), }, }...) - requireEqual(t, want, resApp.resultFloats, "Appended samples not as expected:\n%s", slApp) + requireEqual(t, want, appTest.ResultSamples, "Appended samples not as expected:\n%s", slApp) } func TestScrapeLoopDisableStalenessMarkerInjection(t *testing.T) { - var ( - loopDone = atomic.NewBool(false) - appender = &collectResultAppender{} - scraper = &testScraper{} - app = func(_ context.Context) storage.Appender { return appender } - ) + loopDone := atomic.NewBool(false) - sl := newBasicScrapeLoop(t, context.Background(), scraper, app, 10*time.Millisecond) + sl, scraper, appTest := newTestScrapeLoop(t) scraper.scrapeFunc = func(ctx context.Context, w io.Writer) error { if _, err := w.Write([]byte("metric_a 42\n")); err != nil { return err @@ -5923,9 +5794,7 @@ func TestScrapeLoopDisableStalenessMarkerInjection(t *testing.T) { // Wait for some samples to be appended. require.Eventually(t, func() bool { - appender.mtx.Lock() - defer appender.mtx.Unlock() - return len(appender.resultFloats) > 2 + return appTest.ResultSamplesGreaterThan(2) }, 5*time.Second, 100*time.Millisecond, "Scrape loop didn't append any samples.") // Disable end of run staleness markers and stop the loop. @@ -5936,9 +5805,9 @@ func TestScrapeLoopDisableStalenessMarkerInjection(t *testing.T) { }, 5*time.Second, 100*time.Millisecond, "Scrape loop didn't stop.") // No stale markers should be appended, since they were disabled. - for _, s := range appender.resultFloats { - if value.IsStaleNaN(s.f) { - t.Fatalf("Got stale NaN samples while end of run staleness is disabled: %x", math.Float64bits(s.f)) + for _, s := range appTest.ResultSamples { + if value.IsStaleNaN(s.V) { + t.Fatalf("Got stale NaN samples while end of run staleness is disabled: %x", math.Float64bits(s.V)) } } } diff --git a/scrape/target.go b/scrape/target.go index 2aabff20e2..1d4d860039 100644 --- a/scrape/target.go +++ b/scrape/target.go @@ -24,13 +24,14 @@ import ( "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/config" - "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/model/histogram" - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/model/relabel" "github.com/prometheus/prometheus/model/value" "github.com/prometheus/prometheus/storage" + + "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery/targetgroup" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/model/relabel" ) // TargetHealth describes the health state of a target. @@ -325,13 +326,13 @@ var ( // limitAppender limits the number of total appended samples in a batch. type limitAppender struct { - storage.Appender + storage.AppenderV2 limit int i int } -func (app *limitAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64, v float64) (storage.SeriesRef, error) { +func (app *limitAppender) Append(ref storage.SeriesRef, ls labels.Labels, st, t int64, v float64, h *histogram.Histogram, fh *histogram.FloatHistogram, opts storage.AOptions) (storage.SeriesRef, error) { // Bypass sample_limit checks only if we have a staleness marker for a known series (ref value is non-zero). // This ensures that if a series is already in TSDB then we always write the marker. if ref == 0 || !value.IsStaleNaN(v) { @@ -340,56 +341,31 @@ func (app *limitAppender) Append(ref storage.SeriesRef, lset labels.Labels, t in return 0, errSampleLimit } } - ref, err := app.Appender.Append(ref, lset, t, v) - if err != nil { - return 0, err - } - return ref, nil -} - -func (app *limitAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { - // Bypass sample_limit checks only if we have a staleness marker for a known series (ref value is non-zero). - // This ensures that if a series is already in TSDB then we always write the marker. - if ref == 0 || (h != nil && !value.IsStaleNaN(h.Sum)) || (fh != nil && !value.IsStaleNaN(fh.Sum)) { - app.i++ - if app.i > app.limit { - return 0, errSampleLimit - } - } - ref, err := app.Appender.AppendHistogram(ref, lset, t, h, fh) - if err != nil { - return 0, err - } - return ref, nil + return app.AppenderV2.Append(ref, ls, st, t, v, h, fh, opts) } type timeLimitAppender struct { - storage.Appender + storage.AppenderV2 maxTime int64 } -func (app *timeLimitAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64, v float64) (storage.SeriesRef, error) { +func (app *timeLimitAppender) Append(ref storage.SeriesRef, ls labels.Labels, st, t int64, v float64, h *histogram.Histogram, fh *histogram.FloatHistogram, opts storage.AOptions) (storage.SeriesRef, error) { if t > app.maxTime { return 0, storage.ErrOutOfBounds } - ref, err := app.Appender.Append(ref, lset, t, v) - if err != nil { - return 0, err - } - return ref, nil + return app.AppenderV2.Append(ref, ls, st, t, v, h, fh, opts) } // bucketLimitAppender limits the number of total appended samples in a batch. type bucketLimitAppender struct { - storage.Appender + storage.AppenderV2 limit int } -func (app *bucketLimitAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { - var err error +func (app *bucketLimitAppender) Append(ref storage.SeriesRef, ls labels.Labels, st, t int64, v float64, h *histogram.Histogram, fh *histogram.FloatHistogram, opts storage.AOptions) (_ storage.SeriesRef, err error) { if h != nil { // Return with an early error if the histogram has too many buckets and the // schema is not exponential, in which case we can't reduce the resolution. @@ -420,20 +396,16 @@ func (app *bucketLimitAppender) AppendHistogram(ref storage.SeriesRef, lset labe } } } - if ref, err = app.Appender.AppendHistogram(ref, lset, t, h, fh); err != nil { - return 0, err - } - return ref, nil + return app.AppenderV2.Append(ref, ls, st, t, v, h, fh, opts) } type maxSchemaAppender struct { - storage.Appender + storage.AppenderV2 maxSchema int32 } -func (app *maxSchemaAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { - var err error +func (app *maxSchemaAppender) Append(ref storage.SeriesRef, ls labels.Labels, st, t int64, v float64, h *histogram.Histogram, fh *histogram.FloatHistogram, opts storage.AOptions) (_ storage.SeriesRef, err error) { if h != nil { if histogram.IsExponentialSchemaReserved(h.Schema) && h.Schema > app.maxSchema { if err = h.ReduceResolution(app.maxSchema); err != nil { @@ -448,10 +420,7 @@ func (app *maxSchemaAppender) AppendHistogram(ref storage.SeriesRef, lset labels } } } - if ref, err = app.Appender.AppendHistogram(ref, lset, t, h, fh); err != nil { - return 0, err - } - return ref, nil + return app.AppenderV2.Append(ref, ls, st, t, v, h, fh, opts) } // PopulateDiscoveredLabels sets base labels on lb from target and group labels and scrape configuration, before relabeling. diff --git a/scrape/target_test.go b/scrape/target_test.go index 582b198c79..0adf5dabcb 100644 --- a/scrape/target_test.go +++ b/scrape/target_test.go @@ -14,7 +14,6 @@ package scrape import ( - "context" "crypto/tls" "crypto/x509" "fmt" @@ -611,18 +610,16 @@ func TestBucketLimitAppender(t *testing.T) { }, } - resApp := &collectResultAppender{} - for _, c := range cases { for _, floatHisto := range []bool{true, false} { t.Run(fmt.Sprintf("floatHistogram=%t", floatHisto), func(t *testing.T) { - app := &bucketLimitAppender{Appender: resApp, limit: c.limit} + app := &bucketLimitAppender{AppenderV2: nopAppender{}, limit: c.limit} ts := int64(10 * time.Minute / time.Millisecond) lbls := labels.FromStrings("__name__", "sparse_histogram_series") var err error if floatHisto { fh := c.h.Copy().ToFloat(nil) - _, err = app.AppendHistogram(0, lbls, ts, nil, fh) + _, err = app.Append(0, lbls, 0, ts, 0, nil, fh, storage.AOptions{}) if c.expectError { require.Error(t, err) } else { @@ -632,7 +629,7 @@ func TestBucketLimitAppender(t *testing.T) { } } else { h := c.h.Copy() - _, err = app.AppendHistogram(0, lbls, ts, h, nil) + _, err = app.Append(0, lbls, 0, ts, 0, h, nil, storage.AOptions{}) if c.expectError { require.Error(t, err) } else { @@ -697,23 +694,21 @@ func TestMaxSchemaAppender(t *testing.T) { }, } - resApp := &collectResultAppender{} - for _, c := range cases { for _, floatHisto := range []bool{true, false} { t.Run(fmt.Sprintf("floatHistogram=%t", floatHisto), func(t *testing.T) { - app := &maxSchemaAppender{Appender: resApp, maxSchema: c.maxSchema} + app := &maxSchemaAppender{AppenderV2: nopAppender{}, maxSchema: c.maxSchema} ts := int64(10 * time.Minute / time.Millisecond) lbls := labels.FromStrings("__name__", "sparse_histogram_series") var err error if floatHisto { fh := c.h.Copy().ToFloat(nil) - _, err = app.AppendHistogram(0, lbls, ts, nil, fh) + _, err = app.Append(0, lbls, 0, ts, 0, nil, fh, storage.AOptions{}) require.Equal(t, c.expectSchema, fh.Schema) require.NoError(t, err) } else { h := c.h.Copy() - _, err = app.AppendHistogram(0, lbls, ts, h, nil) + _, err = app.Append(0, lbls, 0, ts, 0, h, nil, storage.AOptions{}) require.Equal(t, c.expectSchema, h.Schema) require.NoError(t, err) } @@ -723,39 +718,37 @@ func TestMaxSchemaAppender(t *testing.T) { } } -// Test sample_limit when a scrape containst Native Histograms. +// Test sample_limit when a scrape contains Native Histograms. func TestAppendWithSampleLimitAndNativeHistogram(t *testing.T) { const sampleLimit = 2 - resApp := &collectResultAppender{} - sl := newBasicScrapeLoop(t, context.Background(), nil, func(_ context.Context) storage.Appender { - return resApp - }, 0) + + sl, _, _ := newTestScrapeLoop(t) sl.sampleLimit = sampleLimit now := time.Now() - app := appender(sl.appender(context.Background()), sl.sampleLimit, sl.bucketLimit, sl.maxSchema) + app := appender(sl.appendableV2.AppenderV2(sl.ctx), sl.sampleLimit, sl.bucketLimit, sl.maxSchema) // sample_limit is set to 2, so first two scrapes should work - _, err := app.Append(0, labels.FromStrings(model.MetricNameLabel, "foo"), timestamp.FromTime(now), 1) - require.NoError(t, err) + { + ls := labels.FromStrings(model.MetricNameLabel, "foo") + ts := timestamp.FromTime(now) + _, err := app.Append(0, ls, 0, ts, 1, nil, nil, storage.AOptions{}) + require.NoError(t, err) + } // Second sample, should be ok. - _, err = app.AppendHistogram( - 0, - labels.FromStrings(model.MetricNameLabel, "my_histogram1"), - timestamp.FromTime(now), - &histogram.Histogram{}, - nil, - ) - require.NoError(t, err) + { + ls := labels.FromStrings(model.MetricNameLabel, "my_histogram1") + ts := timestamp.FromTime(now) + _, err := app.Append(0, ls, 0, ts, 0, &histogram.Histogram{}, nil, storage.AOptions{}) + require.NoError(t, err) + } // This is third sample with sample_limit=2, it should trigger errSampleLimit. - _, err = app.AppendHistogram( - 0, - labels.FromStrings(model.MetricNameLabel, "my_histogram2"), - timestamp.FromTime(now), - &histogram.Histogram{}, - nil, - ) - require.ErrorIs(t, err, errSampleLimit) + { + ls := labels.FromStrings(model.MetricNameLabel, "my_histogram2") + ts := timestamp.FromTime(now) + _, err := app.Append(0, ls, 0, ts, 0, &histogram.Histogram{}, nil, storage.AOptions{}) + require.ErrorIs(t, err, errSampleLimit) + } } diff --git a/storage/fanout.go b/storage/fanout.go index a699a97b02..5a947b9398 100644 --- a/storage/fanout.go +++ b/storage/fanout.go @@ -19,10 +19,8 @@ import ( "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/exemplar" "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/model/metadata" tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" ) @@ -117,11 +115,11 @@ func (f *fanout) ChunkQuerier(mint, maxt int64) (ChunkQuerier, error) { return NewMergeChunkQuerier([]ChunkQuerier{primary}, secondaries, NewCompactingChunkSeriesMerger(ChainedSeriesMerge)), nil } -func (f *fanout) Appender(ctx context.Context) Appender { - primary := f.primary.Appender(ctx) - secondaries := make([]Appender, 0, len(f.secondaries)) +func (f *fanout) AppenderV2(ctx context.Context) AppenderV2 { + primary := f.primary.AppenderV2(ctx) + secondaries := make([]AppenderV2, 0, len(f.secondaries)) for _, storage := range f.secondaries { - secondaries = append(secondaries, storage.Appender(ctx)) + secondaries = append(secondaries, storage.AppenderV2(ctx)) } return &fanoutAppender{ logger: f.logger, @@ -143,98 +141,18 @@ func (f *fanout) Close() error { type fanoutAppender struct { logger *slog.Logger - primary Appender - secondaries []Appender + primary AppenderV2 + secondaries []AppenderV2 } -// SetOptions propagates the hints to both primary and secondary appenders. -func (f *fanoutAppender) SetOptions(opts *AppendOptions) { - if f.primary != nil { - f.primary.SetOptions(opts) - } - for _, appender := range f.secondaries { - appender.SetOptions(opts) - } -} - -func (f *fanoutAppender) Append(ref SeriesRef, l labels.Labels, t int64, v float64) (SeriesRef, error) { - ref, err := f.primary.Append(ref, l, t, v) +func (f *fanoutAppender) Append(ref SeriesRef, ls labels.Labels, st, t int64, v float64, h *histogram.Histogram, fh *histogram.FloatHistogram, opts AppendV2Options) (SeriesRef, error) { + ref, err := f.primary.Append(ref, ls, st, t, v, h, fh, opts) if err != nil { return ref, err } for _, appender := range f.secondaries { - if _, err := appender.Append(ref, l, t, v); err != nil { - return 0, err - } - } - return ref, nil -} - -func (f *fanoutAppender) AppendExemplar(ref SeriesRef, l labels.Labels, e exemplar.Exemplar) (SeriesRef, error) { - ref, err := f.primary.AppendExemplar(ref, l, e) - if err != nil { - return ref, err - } - - for _, appender := range f.secondaries { - if _, err := appender.AppendExemplar(ref, l, e); err != nil { - return 0, err - } - } - return ref, nil -} - -func (f *fanoutAppender) AppendHistogram(ref SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (SeriesRef, error) { - ref, err := f.primary.AppendHistogram(ref, l, t, h, fh) - if err != nil { - return ref, err - } - - for _, appender := range f.secondaries { - if _, err := appender.AppendHistogram(ref, l, t, h, fh); err != nil { - return 0, err - } - } - return ref, nil -} - -func (f *fanoutAppender) AppendHistogramSTZeroSample(ref SeriesRef, l labels.Labels, t, st int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (SeriesRef, error) { - ref, err := f.primary.AppendHistogramSTZeroSample(ref, l, t, st, h, fh) - if err != nil { - return ref, err - } - - for _, appender := range f.secondaries { - if _, err := appender.AppendHistogramSTZeroSample(ref, l, t, st, h, fh); err != nil { - return 0, err - } - } - return ref, nil -} - -func (f *fanoutAppender) UpdateMetadata(ref SeriesRef, l labels.Labels, m metadata.Metadata) (SeriesRef, error) { - ref, err := f.primary.UpdateMetadata(ref, l, m) - if err != nil { - return ref, err - } - - for _, appender := range f.secondaries { - if _, err := appender.UpdateMetadata(ref, l, m); err != nil { - return 0, err - } - } - return ref, nil -} - -func (f *fanoutAppender) AppendSTZeroSample(ref SeriesRef, l labels.Labels, t, st int64) (SeriesRef, error) { - ref, err := f.primary.AppendSTZeroSample(ref, l, t, st) - if err != nil { - return ref, err - } - - for _, appender := range f.secondaries { - if _, err := appender.AppendSTZeroSample(ref, l, t, st); err != nil { + if _, err := appender.Append(ref, ls, st, t, v, h, fh, opts); err != nil { return 0, err } } diff --git a/storage/interface.go b/storage/interface.go index 4011de8ce5..87d7fc8955 100644 --- a/storage/interface.go +++ b/storage/interface.go @@ -80,7 +80,7 @@ type SampleAndChunkQueryable interface { // are goroutine-safe. Storage implements storage.Appender. type Storage interface { SampleAndChunkQueryable - Appendable + AppendableV2 // StartTime returns the oldest timestamp stored in the storage. StartTime() (int64, error) diff --git a/storage/remote/codec_test.go b/storage/remote/codec_test.go index ba67ff33d9..58037f7142 100644 --- a/storage/remote/codec_test.go +++ b/storage/remote/codec_test.go @@ -62,9 +62,10 @@ var ( {Name: "d", Value: "e"}, {Name: "foo", Value: "bar"}, }, - Samples: []prompb.Sample{{Value: 1, Timestamp: 1}}, - Exemplars: []prompb.Exemplar{{Labels: []prompb.Label{{Name: "f", Value: "g"}}, Value: 1, Timestamp: 1}}, - Histograms: []prompb.Histogram{prompb.FromIntHistogram(1, &testHistogram), prompb.FromFloatHistogram(2, testHistogram.ToFloat(nil))}, + Samples: []prompb.Sample{{Value: 1, Timestamp: 1}}, + Exemplars: []prompb.Exemplar{{Labels: []prompb.Label{{Name: "f", Value: "g"}}, Value: 1, Timestamp: 1}}, + // TODO: For RW1 can you send both sample and histogram? (not allowed for RW2). + Histograms: []prompb.Histogram{prompb.FromIntHistogram(2, &testHistogram), prompb.FromFloatHistogram(3, testHistogram.ToFloat(nil))}, }, { Labels: []prompb.Label{ @@ -74,9 +75,10 @@ var ( {Name: "d", Value: "e"}, {Name: "foo", Value: "bar"}, }, - Samples: []prompb.Sample{{Value: 2, Timestamp: 2}}, - Exemplars: []prompb.Exemplar{{Labels: []prompb.Label{{Name: "h", Value: "i"}}, Value: 2, Timestamp: 2}}, - Histograms: []prompb.Histogram{prompb.FromIntHistogram(3, &testHistogram), prompb.FromFloatHistogram(4, testHistogram.ToFloat(nil))}, + Samples: []prompb.Sample{{Value: 2, Timestamp: 4}}, + Exemplars: []prompb.Exemplar{{Labels: []prompb.Label{{Name: "h", Value: "i"}}, Value: 2, Timestamp: 2}}, + // TODO: For RW1 can you send both sample and histogram? (not allowed for RW2). + Histograms: []prompb.Histogram{prompb.FromIntHistogram(5, &testHistogram), prompb.FromFloatHistogram(6, testHistogram.ToFloat(nil))}, }, }, } @@ -90,6 +92,9 @@ var ( Type: model.MetricTypeCounter, Help: "Test counter for test purposes", } + writeV2RequestSeries3Metadata = metadata.Metadata{ + Type: model.MetricTypeHistogram, + } testHistogramCustomBuckets = histogram.Histogram{ Schema: histogram.CustomBucketsSchema, @@ -101,7 +106,7 @@ var ( } // writeV2RequestFixture represents the same request as writeRequestFixture, - // but using the v2 representation, plus includes writeV2RequestSeries1Metadata and writeV2RequestSeries2Metadata. + // but using the v2 representation, plus includes writeV2RequestSeries1Metadata, writeV2RequestSeries2Metadata and writeV2RequestSeries3Metadata. // NOTE: Use TestWriteV2RequestFixture and copy the diff to regenerate if needed. writeV2RequestFixture = &writev2.Request{ Symbols: []string{"", "__name__", "test_metric1", "b", "c", "baz", "qux", "d", "e", "foo", "bar", "f", "g", "h", "i", "Test gauge for test purposes", "Maybe op/sec who knows (:", "Test counter for test purposes"}, @@ -116,12 +121,6 @@ var ( }, Samples: []writev2.Sample{{Value: 1, Timestamp: 10, StartTimestamp: 1}}, // ST needs to be lower than the sample's timestamp. Exemplars: []writev2.Exemplar{{LabelsRefs: []uint32{11, 12}, Value: 1, Timestamp: 10}}, - Histograms: []writev2.Histogram{ - writev2.FromIntHistogram(10, &testHistogram), - writev2.FromFloatHistogram(20, testHistogram.ToFloat(nil)), - writev2.FromIntHistogram(30, &testHistogramCustomBuckets), - writev2.FromFloatHistogram(40, testHistogramCustomBuckets.ToFloat(nil)), - }, }, { LabelsRefs: []uint32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, // Same series as first. @@ -133,11 +132,22 @@ var ( }, Samples: []writev2.Sample{{Value: 2, Timestamp: 20}}, Exemplars: []writev2.Exemplar{{LabelsRefs: []uint32{13, 14}, Value: 2, Timestamp: 20}}, + }, + { + LabelsRefs: []uint32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, // Same series as first two. + Metadata: writev2.Metadata{ + Type: writev2.Metadata_METRIC_TYPE_HISTOGRAM, // writeV2RequestSeries3Metadata.Type. + // Missing help and unit. + }, + Exemplars: []writev2.Exemplar{ + {LabelsRefs: []uint32{11, 12}, Value: 3, Timestamp: 30}, + {LabelsRefs: []uint32{11, 12}, Value: 4, Timestamp: 40}, + }, Histograms: []writev2.Histogram{ - writev2.FromIntHistogram(50, &testHistogram), - writev2.FromFloatHistogram(60, testHistogram.ToFloat(nil)), - writev2.FromIntHistogram(70, &testHistogramCustomBuckets), - writev2.FromFloatHistogram(80, testHistogramCustomBuckets.ToFloat(nil)), + writev2.FromIntHistogram(30, &testHistogram), + writev2.FromFloatHistogram(40, testHistogram.ToFloat(nil)), + writev2.FromIntHistogram(50, &testHistogramCustomBuckets), + writev2.FromFloatHistogram(60, testHistogramCustomBuckets.ToFloat(nil)), }, }, }, @@ -183,12 +193,6 @@ func TestWriteV2RequestFixture(t *testing.T) { }, Samples: []writev2.Sample{{Value: 1, Timestamp: 10, StartTimestamp: 1}}, Exemplars: []writev2.Exemplar{{LabelsRefs: exemplar1LabelRefs, Value: 1, Timestamp: 10}}, - Histograms: []writev2.Histogram{ - writev2.FromIntHistogram(10, &testHistogram), - writev2.FromFloatHistogram(20, testHistogram.ToFloat(nil)), - writev2.FromIntHistogram(30, &testHistogramCustomBuckets), - writev2.FromFloatHistogram(40, testHistogramCustomBuckets.ToFloat(nil)), - }, }, { LabelsRefs: labelRefs, @@ -199,11 +203,22 @@ func TestWriteV2RequestFixture(t *testing.T) { }, Samples: []writev2.Sample{{Value: 2, Timestamp: 20}}, Exemplars: []writev2.Exemplar{{LabelsRefs: exemplar2LabelRefs, Value: 2, Timestamp: 20}}, + }, + { + LabelsRefs: labelRefs, + Metadata: writev2.Metadata{ + Type: writev2.Metadata_METRIC_TYPE_HISTOGRAM, + // No unit, no help. + }, + Exemplars: []writev2.Exemplar{ + {LabelsRefs: exemplar1LabelRefs, Value: 3, Timestamp: 30}, + {LabelsRefs: exemplar1LabelRefs, Value: 4, Timestamp: 40}, + }, Histograms: []writev2.Histogram{ - writev2.FromIntHistogram(50, &testHistogram), - writev2.FromFloatHistogram(60, testHistogram.ToFloat(nil)), - writev2.FromIntHistogram(70, &testHistogramCustomBuckets), - writev2.FromFloatHistogram(80, testHistogramCustomBuckets.ToFloat(nil)), + writev2.FromIntHistogram(30, &testHistogram), + writev2.FromFloatHistogram(40, testHistogram.ToFloat(nil)), + writev2.FromIntHistogram(50, &testHistogramCustomBuckets), + writev2.FromFloatHistogram(60, testHistogramCustomBuckets.ToFloat(nil)), }, }, }, diff --git a/storage/remote/otlptranslator/prometheusremotewrite/combined_appender.go b/storage/remote/otlptranslator/prometheusremotewrite/combined_appender.go deleted file mode 100644 index 883b8d3142..0000000000 --- a/storage/remote/otlptranslator/prometheusremotewrite/combined_appender.go +++ /dev/null @@ -1,244 +0,0 @@ -// Copyright The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// TODO(krajorama): rename this package to otlpappender or similar, as it is -// not specific to Prometheus remote write anymore. -// Note otlptranslator is already used by prometheus/otlptranslator repo. -package prometheusremotewrite - -import ( - "errors" - "fmt" - "log/slog" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - - "github.com/prometheus/prometheus/model/exemplar" - "github.com/prometheus/prometheus/model/histogram" - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/model/metadata" - "github.com/prometheus/prometheus/storage" -) - -// Metadata extends metadata.Metadata with the metric family name. -// OTLP calculates the metric family name for all metrics and uses -// it for generating summary, histogram series by adding the magic -// suffixes. The metric family name is passed down to the appender -// in case the storage needs it for metadata updates. -// Known user is Mimir that implements /api/v1/metadata and uses -// Remote-Write 1.0 for this. Might be removed later if no longer -// needed by any downstream project. -type Metadata struct { - metadata.Metadata - MetricFamilyName string -} - -// CombinedAppender is similar to storage.Appender, but combines updates to -// metadata, created timestamps, exemplars and samples into a single call. -type CombinedAppender interface { - // AppendSample appends a sample and related exemplars, metadata, and - // created timestamp to the storage. - AppendSample(ls labels.Labels, meta Metadata, st, t int64, v float64, es []exemplar.Exemplar) error - // AppendHistogram appends a histogram and related exemplars, metadata, and - // created timestamp to the storage. - AppendHistogram(ls labels.Labels, meta Metadata, st, t int64, h *histogram.Histogram, es []exemplar.Exemplar) error -} - -// CombinedAppenderMetrics is for the metrics observed by the -// combinedAppender implementation. -type CombinedAppenderMetrics struct { - samplesAppendedWithoutMetadata prometheus.Counter - outOfOrderExemplars prometheus.Counter -} - -func NewCombinedAppenderMetrics(reg prometheus.Registerer) CombinedAppenderMetrics { - return CombinedAppenderMetrics{ - samplesAppendedWithoutMetadata: promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Namespace: "prometheus", - Subsystem: "api", - Name: "otlp_appended_samples_without_metadata_total", - Help: "The total number of samples ingested from OTLP without corresponding metadata.", - }), - outOfOrderExemplars: promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Namespace: "prometheus", - Subsystem: "api", - Name: "otlp_out_of_order_exemplars_total", - Help: "The total number of received OTLP exemplars which were rejected because they were out of order.", - }), - } -} - -// NewCombinedAppender creates a combined appender that sets start times and -// updates metadata for each series only once, and appends samples and -// exemplars for each call. -func NewCombinedAppender(app storage.Appender, logger *slog.Logger, ingestSTZeroSample, appendMetadata bool, metrics CombinedAppenderMetrics) CombinedAppender { - return &combinedAppender{ - app: app, - logger: logger, - ingestSTZeroSample: ingestSTZeroSample, - appendMetadata: appendMetadata, - refs: make(map[uint64]seriesRef), - samplesAppendedWithoutMetadata: metrics.samplesAppendedWithoutMetadata, - outOfOrderExemplars: metrics.outOfOrderExemplars, - } -} - -type seriesRef struct { - ref storage.SeriesRef - st int64 - ls labels.Labels - meta metadata.Metadata -} - -type combinedAppender struct { - app storage.Appender - logger *slog.Logger - samplesAppendedWithoutMetadata prometheus.Counter - outOfOrderExemplars prometheus.Counter - ingestSTZeroSample bool - appendMetadata bool - // Used to ensure we only update metadata and created timestamps once, and to share storage.SeriesRefs. - // To detect hash collision it also stores the labels. - // There is no overflow/conflict list, the TSDB will handle that part. - refs map[uint64]seriesRef -} - -func (b *combinedAppender) AppendSample(ls labels.Labels, meta Metadata, st, t int64, v float64, es []exemplar.Exemplar) (err error) { - return b.appendFloatOrHistogram(ls, meta.Metadata, st, t, v, nil, es) -} - -func (b *combinedAppender) AppendHistogram(ls labels.Labels, meta Metadata, st, t int64, h *histogram.Histogram, es []exemplar.Exemplar) (err error) { - if h == nil { - // Sanity check, we should never get here with a nil histogram. - b.logger.Error("Received nil histogram in CombinedAppender.AppendHistogram", "series", ls.String()) - return errors.New("internal error, attempted to append nil histogram") - } - return b.appendFloatOrHistogram(ls, meta.Metadata, st, t, 0, h, es) -} - -func (b *combinedAppender) appendFloatOrHistogram(ls labels.Labels, meta metadata.Metadata, st, t int64, v float64, h *histogram.Histogram, es []exemplar.Exemplar) (err error) { - hash := ls.Hash() - series, exists := b.refs[hash] - ref := series.ref - if exists && !labels.Equal(series.ls, ls) { - // Hash collision. The series reference we stored is pointing to a - // different series so we cannot use it, we need to reset the - // reference and cache. - // Note: we don't need to keep track of conflicts here, - // the TSDB will handle that part when we pass 0 reference. - exists = false - ref = 0 - } - updateRefs := !exists || series.st != st - if updateRefs && st != 0 && st < t && b.ingestSTZeroSample { - var newRef storage.SeriesRef - if h != nil { - newRef, err = b.app.AppendHistogramSTZeroSample(ref, ls, t, st, h, nil) - } else { - newRef, err = b.app.AppendSTZeroSample(ref, ls, t, st) - } - if err != nil { - if !errors.Is(err, storage.ErrOutOfOrderST) && !errors.Is(err, storage.ErrDuplicateSampleForTimestamp) { - // Even for the first sample OOO is a common scenario because - // we can't tell if a ST was already ingested in a previous request. - // We ignore the error. - // ErrDuplicateSampleForTimestamp is also a common scenario because - // unknown start times in Opentelemetry are indicated by setting - // the start time to the same as the first sample time. - // https://opentelemetry.io/docs/specs/otel/metrics/data-model/#cumulative-streams-handling-unknown-start-time - b.logger.Warn("Error when appending ST from OTLP", "err", err, "series", ls.String(), "start_timestamp", st, "timestamp", t, "sample_type", sampleType(h)) - } - } else { - // We only use the returned reference on success as otherwise an - // error of ST append could invalidate the series reference. - ref = newRef - } - } - { - var newRef storage.SeriesRef - if h != nil { - newRef, err = b.app.AppendHistogram(ref, ls, t, h, nil) - } else { - newRef, err = b.app.Append(ref, ls, t, v) - } - if err != nil { - // Although Append does not currently return ErrDuplicateSampleForTimestamp there is - // a note indicating its inclusion in the future. - if errors.Is(err, storage.ErrOutOfOrderSample) || - errors.Is(err, storage.ErrOutOfBounds) || - errors.Is(err, storage.ErrDuplicateSampleForTimestamp) { - b.logger.Error("Error when appending sample from OTLP", "err", err.Error(), "series", ls.String(), "timestamp", t, "sample_type", sampleType(h)) - } - } else { - // If the append was successful, we can use the returned reference. - ref = newRef - } - } - - if ref == 0 { - // We cannot update metadata or add exemplars on non existent series. - return err - } - - metadataChanged := exists && (series.meta.Help != meta.Help || series.meta.Type != meta.Type || series.meta.Unit != meta.Unit) - - // Update cache if references changed or metadata changed. - if updateRefs || metadataChanged { - b.refs[hash] = seriesRef{ - ref: ref, - st: st, - ls: ls, - meta: meta, - } - } - - // Update metadata in storage if enabled and needed. - if b.appendMetadata && (!exists || metadataChanged) { - // Only update metadata in WAL if the metadata-wal-records feature is enabled. - // Without this feature, metadata is not persisted to WAL. - _, err := b.app.UpdateMetadata(ref, ls, meta) - if err != nil { - b.samplesAppendedWithoutMetadata.Add(1) - b.logger.Warn("Error while updating metadata from OTLP", "err", err) - } - } - - b.appendExemplars(ref, ls, es) - - return err -} - -func sampleType(h *histogram.Histogram) string { - if h == nil { - return "float" - } - return "histogram" -} - -func (b *combinedAppender) appendExemplars(ref storage.SeriesRef, ls labels.Labels, es []exemplar.Exemplar) storage.SeriesRef { - var err error - for _, e := range es { - if ref, err = b.app.AppendExemplar(ref, ls, e); err != nil { - switch { - case errors.Is(err, storage.ErrOutOfOrderExemplar): - b.outOfOrderExemplars.Add(1) - b.logger.Debug("Out of order exemplar from OTLP", "series", ls.String(), "exemplar", fmt.Sprintf("%+v", e)) - default: - // Since exemplar storage is still experimental, we don't fail the request on ingestion errors - b.logger.Debug("Error while adding exemplar from OTLP", "series", ls.String(), "exemplar", fmt.Sprintf("%+v", e), "err", err) - } - } - } - return ref -} diff --git a/storage/remote/otlptranslator/prometheusremotewrite/combined_appender_test.go b/storage/remote/otlptranslator/prometheusremotewrite/combined_appender_test.go deleted file mode 100644 index 753112cf82..0000000000 --- a/storage/remote/otlptranslator/prometheusremotewrite/combined_appender_test.go +++ /dev/null @@ -1,937 +0,0 @@ -// Copyright 2025 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package prometheusremotewrite - -import ( - "bytes" - "context" - "errors" - "fmt" - "math" - "testing" - "time" - - "github.com/google/go-cmp/cmp" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/common/model" - "github.com/prometheus/common/promslog" - "github.com/stretchr/testify/require" - - "github.com/prometheus/prometheus/model/exemplar" - "github.com/prometheus/prometheus/model/histogram" - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/model/metadata" - "github.com/prometheus/prometheus/storage" - "github.com/prometheus/prometheus/tsdb" - "github.com/prometheus/prometheus/tsdb/chunkenc" - "github.com/prometheus/prometheus/tsdb/tsdbutil" - "github.com/prometheus/prometheus/util/testutil" -) - -type mockCombinedAppender struct { - pendingSamples []combinedSample - pendingHistograms []combinedHistogram - - samples []combinedSample - histograms []combinedHistogram -} - -type combinedSample struct { - metricFamilyName string - ls labels.Labels - meta metadata.Metadata - t int64 - st int64 - v float64 - es []exemplar.Exemplar -} - -type combinedHistogram struct { - metricFamilyName string - ls labels.Labels - meta metadata.Metadata - t int64 - st int64 - h *histogram.Histogram - es []exemplar.Exemplar -} - -func (m *mockCombinedAppender) AppendSample(ls labels.Labels, meta Metadata, st, t int64, v float64, es []exemplar.Exemplar) error { - m.pendingSamples = append(m.pendingSamples, combinedSample{ - metricFamilyName: meta.MetricFamilyName, - ls: ls, - meta: meta.Metadata, - t: t, - st: st, - v: v, - es: es, - }) - return nil -} - -func (m *mockCombinedAppender) AppendHistogram(ls labels.Labels, meta Metadata, st, t int64, h *histogram.Histogram, es []exemplar.Exemplar) error { - m.pendingHistograms = append(m.pendingHistograms, combinedHistogram{ - metricFamilyName: meta.MetricFamilyName, - ls: ls, - meta: meta.Metadata, - t: t, - st: st, - h: h, - es: es, - }) - return nil -} - -func (m *mockCombinedAppender) Commit() error { - m.samples = append(m.samples, m.pendingSamples...) - m.pendingSamples = m.pendingSamples[:0] - m.histograms = append(m.histograms, m.pendingHistograms...) - m.pendingHistograms = m.pendingHistograms[:0] - return nil -} - -func requireEqual(t testing.TB, expected, actual any, msgAndArgs ...any) { - testutil.RequireEqualWithOptions(t, expected, actual, []cmp.Option{cmp.AllowUnexported(combinedSample{}, combinedHistogram{})}, msgAndArgs...) -} - -// TestCombinedAppenderOnTSDB runs some basic tests on a real TSDB to check -// that the combinedAppender works on a real TSDB. -func TestCombinedAppenderOnTSDB(t *testing.T) { - t.Run("ingestSTZeroSample=false", func(t *testing.T) { testCombinedAppenderOnTSDB(t, false) }) - - t.Run("ingestSTZeroSample=true", func(t *testing.T) { testCombinedAppenderOnTSDB(t, true) }) -} - -func testCombinedAppenderOnTSDB(t *testing.T, ingestSTZeroSample bool) { - t.Helper() - - now := time.Now() - - testExemplars := []exemplar.Exemplar{ - { - Labels: labels.FromStrings("tracid", "122"), - Value: 1337, - }, - { - Labels: labels.FromStrings("tracid", "132"), - Value: 7777, - }, - } - expectedExemplars := []exemplar.QueryResult{ - { - SeriesLabels: labels.FromStrings( - model.MetricNameLabel, "test_bytes_total", - "foo", "bar", - ), - Exemplars: testExemplars, - }, - } - - seriesLabels := labels.FromStrings( - model.MetricNameLabel, "test_bytes_total", - "foo", "bar", - ) - floatMetadata := Metadata{ - Metadata: metadata.Metadata{ - Type: model.MetricTypeCounter, - Unit: "bytes", - Help: "some help", - }, - MetricFamilyName: "test_bytes_total", - } - - histogramMetadata := Metadata{ - Metadata: metadata.Metadata{ - Type: model.MetricTypeHistogram, - Unit: "bytes", - Help: "some help", - }, - MetricFamilyName: "test_bytes", - } - - testCases := map[string]struct { - appendFunc func(*testing.T, CombinedAppender) - extraAppendFunc func(*testing.T, CombinedAppender) - expectedSamples []sample - expectedExemplars []exemplar.QueryResult - expectedLogsForST []string - }{ - "single float sample, zero ST": { - appendFunc: func(t *testing.T, app CombinedAppender) { - require.NoError(t, app.AppendSample(seriesLabels.Copy(), floatMetadata, 0, now.UnixMilli(), 42.0, testExemplars)) - }, - expectedSamples: []sample{ - { - t: now.UnixMilli(), - f: 42.0, - }, - }, - expectedExemplars: expectedExemplars, - }, - "single float sample, very old ST": { - appendFunc: func(t *testing.T, app CombinedAppender) { - require.NoError(t, app.AppendSample(seriesLabels.Copy(), floatMetadata, 1, now.UnixMilli(), 42.0, nil)) - }, - expectedSamples: []sample{ - { - t: now.UnixMilli(), - f: 42.0, - }, - }, - expectedLogsForST: []string{ - "Error when appending ST from OTLP", - "out of bound", - }, - }, - "single float sample, normal ST": { - appendFunc: func(t *testing.T, app CombinedAppender) { - require.NoError(t, app.AppendSample(seriesLabels.Copy(), floatMetadata, now.Add(-2*time.Minute).UnixMilli(), now.UnixMilli(), 42.0, nil)) - }, - expectedSamples: []sample{ - { - stZero: true, - t: now.Add(-2 * time.Minute).UnixMilli(), - }, - { - t: now.UnixMilli(), - f: 42.0, - }, - }, - }, - "single float sample, ST same time as sample": { - appendFunc: func(t *testing.T, app CombinedAppender) { - require.NoError(t, app.AppendSample(seriesLabels.Copy(), floatMetadata, now.UnixMilli(), now.UnixMilli(), 42.0, nil)) - }, - expectedSamples: []sample{ - { - t: now.UnixMilli(), - f: 42.0, - }, - }, - }, - "two float samples in different messages, ST same time as first sample": { - appendFunc: func(t *testing.T, app CombinedAppender) { - require.NoError(t, app.AppendSample(seriesLabels.Copy(), floatMetadata, now.UnixMilli(), now.UnixMilli(), 42.0, nil)) - }, - extraAppendFunc: func(t *testing.T, app CombinedAppender) { - require.NoError(t, app.AppendSample(seriesLabels.Copy(), floatMetadata, now.UnixMilli(), now.Add(time.Second).UnixMilli(), 43.0, nil)) - }, - expectedSamples: []sample{ - { - t: now.UnixMilli(), - f: 42.0, - }, - { - t: now.Add(time.Second).UnixMilli(), - f: 43.0, - }, - }, - }, - "single float sample, ST in the future of the sample": { - appendFunc: func(t *testing.T, app CombinedAppender) { - require.NoError(t, app.AppendSample(seriesLabels.Copy(), floatMetadata, now.Add(time.Minute).UnixMilli(), now.UnixMilli(), 42.0, nil)) - }, - expectedSamples: []sample{ - { - t: now.UnixMilli(), - f: 42.0, - }, - }, - }, - "single histogram sample, zero ST": { - appendFunc: func(t *testing.T, app CombinedAppender) { - require.NoError(t, app.AppendHistogram(seriesLabels.Copy(), histogramMetadata, 0, now.UnixMilli(), tsdbutil.GenerateTestHistogram(42), testExemplars)) - }, - expectedSamples: []sample{ - { - t: now.UnixMilli(), - h: tsdbutil.GenerateTestHistogram(42), - }, - }, - expectedExemplars: expectedExemplars, - }, - "single histogram sample, very old ST": { - appendFunc: func(t *testing.T, app CombinedAppender) { - require.NoError(t, app.AppendHistogram(seriesLabels.Copy(), histogramMetadata, 1, now.UnixMilli(), tsdbutil.GenerateTestHistogram(42), nil)) - }, - expectedSamples: []sample{ - { - t: now.UnixMilli(), - h: tsdbutil.GenerateTestHistogram(42), - }, - }, - expectedLogsForST: []string{ - "Error when appending ST from OTLP", - "out of bound", - }, - }, - "single histogram sample, normal ST": { - appendFunc: func(t *testing.T, app CombinedAppender) { - require.NoError(t, app.AppendHistogram(seriesLabels.Copy(), histogramMetadata, now.Add(-2*time.Minute).UnixMilli(), now.UnixMilli(), tsdbutil.GenerateTestHistogram(42), nil)) - }, - expectedSamples: []sample{ - { - stZero: true, - t: now.Add(-2 * time.Minute).UnixMilli(), - h: &histogram.Histogram{}, - }, - { - t: now.UnixMilli(), - h: tsdbutil.GenerateTestHistogram(42), - }, - }, - }, - "single histogram sample, ST same time as sample": { - appendFunc: func(t *testing.T, app CombinedAppender) { - require.NoError(t, app.AppendHistogram(seriesLabels.Copy(), histogramMetadata, now.UnixMilli(), now.UnixMilli(), tsdbutil.GenerateTestHistogram(42), nil)) - }, - expectedSamples: []sample{ - { - t: now.UnixMilli(), - h: tsdbutil.GenerateTestHistogram(42), - }, - }, - }, - "two histogram samples in different messages, ST same time as first sample": { - appendFunc: func(t *testing.T, app CombinedAppender) { - require.NoError(t, app.AppendHistogram(seriesLabels.Copy(), floatMetadata, now.UnixMilli(), now.UnixMilli(), tsdbutil.GenerateTestHistogram(42), nil)) - }, - extraAppendFunc: func(t *testing.T, app CombinedAppender) { - require.NoError(t, app.AppendHistogram(seriesLabels.Copy(), floatMetadata, now.UnixMilli(), now.Add(time.Second).UnixMilli(), tsdbutil.GenerateTestHistogram(43), nil)) - }, - expectedSamples: []sample{ - { - t: now.UnixMilli(), - h: tsdbutil.GenerateTestHistogram(42), - }, - { - t: now.Add(time.Second).UnixMilli(), - h: tsdbutil.GenerateTestHistogram(43), - }, - }, - }, - "single histogram sample, ST in the future of the sample": { - appendFunc: func(t *testing.T, app CombinedAppender) { - require.NoError(t, app.AppendHistogram(seriesLabels.Copy(), histogramMetadata, now.Add(time.Minute).UnixMilli(), now.UnixMilli(), tsdbutil.GenerateTestHistogram(42), nil)) - }, - expectedSamples: []sample{ - { - t: now.UnixMilli(), - h: tsdbutil.GenerateTestHistogram(42), - }, - }, - }, - "multiple float samples": { - appendFunc: func(t *testing.T, app CombinedAppender) { - require.NoError(t, app.AppendSample(seriesLabels.Copy(), floatMetadata, 0, now.UnixMilli(), 42.0, nil)) - require.NoError(t, app.AppendSample(seriesLabels.Copy(), floatMetadata, 0, now.Add(15*time.Second).UnixMilli(), 62.0, nil)) - }, - expectedSamples: []sample{ - { - t: now.UnixMilli(), - f: 42.0, - }, - { - t: now.Add(15 * time.Second).UnixMilli(), - f: 62.0, - }, - }, - }, - "multiple histogram samples": { - appendFunc: func(t *testing.T, app CombinedAppender) { - require.NoError(t, app.AppendHistogram(seriesLabels.Copy(), histogramMetadata, 0, now.UnixMilli(), tsdbutil.GenerateTestHistogram(42), nil)) - require.NoError(t, app.AppendHistogram(seriesLabels.Copy(), histogramMetadata, 0, now.Add(15*time.Second).UnixMilli(), tsdbutil.GenerateTestHistogram(62), nil)) - }, - expectedSamples: []sample{ - { - t: now.UnixMilli(), - h: tsdbutil.GenerateTestHistogram(42), - }, - { - t: now.Add(15 * time.Second).UnixMilli(), - h: tsdbutil.GenerateTestHistogram(62), - }, - }, - }, - "float samples with ST changing": { - appendFunc: func(t *testing.T, app CombinedAppender) { - require.NoError(t, app.AppendSample(seriesLabels.Copy(), floatMetadata, now.Add(-4*time.Second).UnixMilli(), now.Add(-3*time.Second).UnixMilli(), 42.0, nil)) - require.NoError(t, app.AppendSample(seriesLabels.Copy(), floatMetadata, now.Add(-1*time.Second).UnixMilli(), now.UnixMilli(), 62.0, nil)) - }, - expectedSamples: []sample{ - { - stZero: true, - t: now.Add(-4 * time.Second).UnixMilli(), - }, - { - t: now.Add(-3 * time.Second).UnixMilli(), - f: 42.0, - }, - { - stZero: true, - t: now.Add(-1 * time.Second).UnixMilli(), - }, - { - t: now.UnixMilli(), - f: 62.0, - }, - }, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - var expectedLogs []string - if ingestSTZeroSample { - expectedLogs = append(expectedLogs, tc.expectedLogsForST...) - } - - dir := t.TempDir() - opts := tsdb.DefaultOptions() - opts.EnableExemplarStorage = true - opts.MaxExemplars = 100 - db, err := tsdb.Open(dir, promslog.NewNopLogger(), prometheus.NewRegistry(), opts, nil) - require.NoError(t, err) - - t.Cleanup(func() { db.Close() }) - - var output bytes.Buffer - logger := promslog.New(&promslog.Config{Writer: &output}) - - ctx := context.Background() - reg := prometheus.NewRegistry() - cappMetrics := NewCombinedAppenderMetrics(reg) - app := db.Appender(ctx) - capp := NewCombinedAppender(app, logger, ingestSTZeroSample, false, cappMetrics) - tc.appendFunc(t, capp) - require.NoError(t, app.Commit()) - - if tc.extraAppendFunc != nil { - app = db.Appender(ctx) - capp = NewCombinedAppender(app, logger, ingestSTZeroSample, false, cappMetrics) - tc.extraAppendFunc(t, capp) - require.NoError(t, app.Commit()) - } - - if len(expectedLogs) > 0 { - for _, expectedLog := range expectedLogs { - require.Contains(t, output.String(), expectedLog) - } - } else { - require.Empty(t, output.String(), "unexpected log output") - } - - q, err := db.Querier(int64(math.MinInt64), int64(math.MaxInt64)) - require.NoError(t, err) - - ss := q.Select(ctx, false, &storage.SelectHints{ - Start: int64(math.MinInt64), - End: int64(math.MaxInt64), - }, labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test_bytes_total")) - - require.NoError(t, ss.Err()) - - require.True(t, ss.Next()) - series := ss.At() - it := series.Iterator(nil) - for i, sample := range tc.expectedSamples { - if !ingestSTZeroSample && sample.stZero { - continue - } - if sample.h == nil { - require.Equal(t, chunkenc.ValFloat, it.Next()) - ts, v := it.At() - require.Equal(t, sample.t, ts, "sample ts %d", i) - require.Equal(t, sample.f, v, "sample v %d", i) - } else { - require.Equal(t, chunkenc.ValHistogram, it.Next()) - ts, h := it.AtHistogram(nil) - require.Equal(t, sample.t, ts, "sample ts %d", i) - require.Equal(t, sample.h.Count, h.Count, "sample v %d", i) - } - } - require.False(t, ss.Next()) - - eq, err := db.ExemplarQuerier(ctx) - require.NoError(t, err) - exResult, err := eq.Select(int64(math.MinInt64), int64(math.MaxInt64), []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test_bytes_total")}) - require.NoError(t, err) - if tc.expectedExemplars == nil { - tc.expectedExemplars = []exemplar.QueryResult{} - } - require.Equal(t, tc.expectedExemplars, exResult) - }) - } -} - -type sample struct { - stZero bool - - t int64 - f float64 - h *histogram.Histogram -} - -// TestCombinedAppenderSeriesRefs checks that the combined appender -// correctly uses and updates the series references in the internal map. -func TestCombinedAppenderSeriesRefs(t *testing.T) { - seriesLabels := labels.FromStrings( - model.MetricNameLabel, "test_bytes_total", - "foo", "bar", - ) - - floatMetadata := Metadata{ - Metadata: metadata.Metadata{ - Type: model.MetricTypeCounter, - Unit: "bytes", - Help: "some help", - }, - MetricFamilyName: "test_bytes_total", - } - - t.Run("happy case with ST zero, reference is passed and reused", func(t *testing.T) { - app := &appenderRecorder{} - capp := NewCombinedAppender(app, promslog.NewNopLogger(), true, false, NewCombinedAppenderMetrics(prometheus.NewRegistry())) - - require.NoError(t, capp.AppendSample(seriesLabels.Copy(), floatMetadata, 1, 2, 42.0, nil)) - - require.NoError(t, capp.AppendSample(seriesLabels.Copy(), floatMetadata, 3, 4, 62.0, []exemplar.Exemplar{ - { - Labels: labels.FromStrings("tracid", "122"), - Value: 1337, - }, - })) - - require.Len(t, app.records, 5) - requireEqualOpAndRef(t, "AppendSTZeroSample", 0, app.records[0]) - ref := app.records[0].outRef - require.NotZero(t, ref) - requireEqualOpAndRef(t, "Append", ref, app.records[1]) - requireEqualOpAndRef(t, "AppendSTZeroSample", ref, app.records[2]) - requireEqualOpAndRef(t, "Append", ref, app.records[3]) - requireEqualOpAndRef(t, "AppendExemplar", ref, app.records[4]) - }) - - t.Run("error on second ST ingest doesn't update the reference", func(t *testing.T) { - app := &appenderRecorder{} - capp := NewCombinedAppender(app, promslog.NewNopLogger(), true, false, NewCombinedAppenderMetrics(prometheus.NewRegistry())) - - require.NoError(t, capp.AppendSample(seriesLabels.Copy(), floatMetadata, 1, 2, 42.0, nil)) - - app.appendSTZeroSampleError = errors.New("test error") - require.NoError(t, capp.AppendSample(seriesLabels.Copy(), floatMetadata, 3, 4, 62.0, nil)) - - require.Len(t, app.records, 4) - requireEqualOpAndRef(t, "AppendSTZeroSample", 0, app.records[0]) - ref := app.records[0].outRef - require.NotZero(t, ref) - requireEqualOpAndRef(t, "Append", ref, app.records[1]) - requireEqualOpAndRef(t, "AppendSTZeroSample", ref, app.records[2]) - require.Zero(t, app.records[2].outRef, "the second AppendSTZeroSample returned 0") - requireEqualOpAndRef(t, "Append", ref, app.records[3]) - }) - - t.Run("metadata, exemplars are not updated if append failed", func(t *testing.T) { - app := &appenderRecorder{} - capp := NewCombinedAppender(app, promslog.NewNopLogger(), true, false, NewCombinedAppenderMetrics(prometheus.NewRegistry())) - app.appendError = errors.New("test error") - require.Error(t, capp.AppendSample(seriesLabels.Copy(), floatMetadata, 0, 1, 42.0, []exemplar.Exemplar{ - { - Labels: labels.FromStrings("tracid", "122"), - Value: 1337, - }, - })) - - require.Len(t, app.records, 1) - require.Equal(t, appenderRecord{ - op: "Append", - ls: labels.FromStrings(model.MetricNameLabel, "test_bytes_total", "foo", "bar"), - }, app.records[0]) - }) - - t.Run("metadata, exemplars are updated if append failed but reference is valid", func(t *testing.T) { - app := &appenderRecorder{} - capp := NewCombinedAppender(app, promslog.NewNopLogger(), true, true, NewCombinedAppenderMetrics(prometheus.NewRegistry())) - - newMetadata := floatMetadata - newMetadata.Help = "some other help" - - require.NoError(t, capp.AppendSample(seriesLabels.Copy(), floatMetadata, 1, 2, 42.0, nil)) - app.appendError = errors.New("test error") - require.Error(t, capp.AppendSample(seriesLabels.Copy(), newMetadata, 3, 4, 62.0, []exemplar.Exemplar{ - { - Labels: labels.FromStrings("tracid", "122"), - Value: 1337, - }, - })) - - require.Len(t, app.records, 7) - requireEqualOpAndRef(t, "AppendSTZeroSample", 0, app.records[0]) - ref := app.records[0].outRef - require.NotZero(t, ref) - requireEqualOpAndRef(t, "Append", ref, app.records[1]) - requireEqualOpAndRef(t, "UpdateMetadata", ref, app.records[2]) - requireEqualOpAndRef(t, "AppendSTZeroSample", ref, app.records[3]) - requireEqualOpAndRef(t, "Append", ref, app.records[4]) - require.Zero(t, app.records[4].outRef, "the second Append returned 0") - requireEqualOpAndRef(t, "UpdateMetadata", ref, app.records[5]) - requireEqualOpAndRef(t, "AppendExemplar", ref, app.records[6]) - }) - - t.Run("simulate conflict with existing series", func(t *testing.T) { - app := &appenderRecorder{} - capp := NewCombinedAppender(app, promslog.NewNopLogger(), true, false, NewCombinedAppenderMetrics(prometheus.NewRegistry())) - - ls := labels.FromStrings( - model.MetricNameLabel, "test_bytes_total", - "foo", "bar", - ) - - require.NoError(t, capp.AppendSample(ls, floatMetadata, 1, 2, 42.0, nil)) - - hash := ls.Hash() - cappImpl := capp.(*combinedAppender) - series := cappImpl.refs[hash] - series.ls = labels.FromStrings( - model.MetricNameLabel, "test_bytes_total", - "foo", "club", - ) - // The hash and ref remain the same, but we altered the labels. - // This simulates a conflict with an existing series. - cappImpl.refs[hash] = series - - require.NoError(t, capp.AppendSample(ls, floatMetadata, 3, 4, 62.0, []exemplar.Exemplar{ - { - Labels: labels.FromStrings("tracid", "122"), - Value: 1337, - }, - })) - - require.Len(t, app.records, 5) - requireEqualOpAndRef(t, "AppendSTZeroSample", 0, app.records[0]) - ref := app.records[0].outRef - require.NotZero(t, ref) - requireEqualOpAndRef(t, "Append", ref, app.records[1]) - requireEqualOpAndRef(t, "AppendSTZeroSample", 0, app.records[2]) - newRef := app.records[2].outRef - require.NotEqual(t, ref, newRef, "the second AppendSTZeroSample returned a different reference") - requireEqualOpAndRef(t, "Append", newRef, app.records[3]) - requireEqualOpAndRef(t, "AppendExemplar", newRef, app.records[4]) - }) - - t.Run("check that invoking AppendHistogram returns an error for nil histogram", func(t *testing.T) { - app := &appenderRecorder{} - capp := NewCombinedAppender(app, promslog.NewNopLogger(), true, false, NewCombinedAppenderMetrics(prometheus.NewRegistry())) - - ls := labels.FromStrings( - model.MetricNameLabel, "test_bytes_total", - "foo", "bar", - ) - err := capp.AppendHistogram(ls, Metadata{}, 4, 2, nil, nil) - require.Error(t, err) - }) - - for _, appendMetadata := range []bool{false, true} { - t.Run(fmt.Sprintf("appendMetadata=%t", appendMetadata), func(t *testing.T) { - app := &appenderRecorder{} - capp := NewCombinedAppender(app, promslog.NewNopLogger(), true, appendMetadata, NewCombinedAppenderMetrics(prometheus.NewRegistry())) - - require.NoError(t, capp.AppendSample(seriesLabels.Copy(), floatMetadata, 1, 2, 42.0, nil)) - - if appendMetadata { - require.Len(t, app.records, 3) - requireEqualOp(t, "AppendSTZeroSample", app.records[0]) - requireEqualOp(t, "Append", app.records[1]) - requireEqualOp(t, "UpdateMetadata", app.records[2]) - } else { - require.Len(t, app.records, 2) - requireEqualOp(t, "AppendSTZeroSample", app.records[0]) - requireEqualOp(t, "Append", app.records[1]) - } - }) - } -} - -// TestCombinedAppenderMetadataChanges verifies that UpdateMetadata is called -// when metadata fields change (help, unit, or type). -func TestCombinedAppenderMetadataChanges(t *testing.T) { - seriesLabels := labels.FromStrings( - model.MetricNameLabel, "test_metric", - "foo", "bar", - ) - - baseMetadata := Metadata{ - Metadata: metadata.Metadata{ - Type: model.MetricTypeCounter, - Unit: "bytes", - Help: "original help", - }, - MetricFamilyName: "test_metric", - } - - tests := []struct { - name string - modifyMetadata func(Metadata) Metadata - }{ - { - name: "help changes", - modifyMetadata: func(m Metadata) Metadata { - m.Help = "new help text" - return m - }, - }, - { - name: "unit changes", - modifyMetadata: func(m Metadata) Metadata { - m.Unit = "seconds" - return m - }, - }, - { - name: "type changes", - modifyMetadata: func(m Metadata) Metadata { - m.Type = model.MetricTypeGauge - return m - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - app := &appenderRecorder{} - capp := NewCombinedAppender(app, promslog.NewNopLogger(), true, true, NewCombinedAppenderMetrics(prometheus.NewRegistry())) - - newMetadata := tt.modifyMetadata(baseMetadata) - - require.NoError(t, capp.AppendSample(seriesLabels.Copy(), baseMetadata, 1, 2, 42.0, nil)) - require.NoError(t, capp.AppendSample(seriesLabels.Copy(), newMetadata, 3, 4, 62.0, nil)) - require.NoError(t, capp.AppendSample(seriesLabels.Copy(), newMetadata, 3, 5, 162.0, nil)) - - // Verify expected operations. - require.Len(t, app.records, 7) - requireEqualOpAndRef(t, "AppendSTZeroSample", 0, app.records[0]) - ref := app.records[0].outRef - require.NotZero(t, ref) - requireEqualOpAndRef(t, "Append", ref, app.records[1]) - requireEqualOpAndRef(t, "UpdateMetadata", ref, app.records[2]) - requireEqualOpAndRef(t, "AppendSTZeroSample", ref, app.records[3]) - requireEqualOpAndRef(t, "Append", ref, app.records[4]) - requireEqualOpAndRef(t, "UpdateMetadata", ref, app.records[5]) - requireEqualOpAndRef(t, "Append", ref, app.records[6]) - }) - } -} - -func requireEqualOp(t *testing.T, expectedOp string, actual appenderRecord) { - t.Helper() - require.Equal(t, expectedOp, actual.op) -} - -func requireEqualOpAndRef(t *testing.T, expectedOp string, expectedRef storage.SeriesRef, actual appenderRecord) { - t.Helper() - require.Equal(t, expectedOp, actual.op) - require.Equal(t, expectedRef, actual.ref) -} - -type appenderRecord struct { - op string - ref storage.SeriesRef - outRef storage.SeriesRef - ls labels.Labels -} - -type appenderRecorder struct { - refcount uint64 - records []appenderRecord - - appendError error - appendSTZeroSampleError error - appendHistogramError error - appendHistogramSTZeroSampleError error - updateMetadataError error - appendExemplarError error -} - -var _ storage.Appender = &appenderRecorder{} - -func (a *appenderRecorder) setOutRef(ref storage.SeriesRef) { - if len(a.records) == 0 { - return - } - a.records[len(a.records)-1].outRef = ref -} - -func (a *appenderRecorder) newRef() storage.SeriesRef { - a.refcount++ - return storage.SeriesRef(a.refcount) -} - -func (a *appenderRecorder) Append(ref storage.SeriesRef, ls labels.Labels, _ int64, _ float64) (storage.SeriesRef, error) { - a.records = append(a.records, appenderRecord{op: "Append", ref: ref, ls: ls}) - if a.appendError != nil { - return 0, a.appendError - } - if ref == 0 { - ref = a.newRef() - } - a.setOutRef(ref) - return ref, nil -} - -func (a *appenderRecorder) AppendSTZeroSample(ref storage.SeriesRef, ls labels.Labels, _, _ int64) (storage.SeriesRef, error) { - a.records = append(a.records, appenderRecord{op: "AppendSTZeroSample", ref: ref, ls: ls}) - if a.appendSTZeroSampleError != nil { - return 0, a.appendSTZeroSampleError - } - if ref == 0 { - ref = a.newRef() - } - a.setOutRef(ref) - return ref, nil -} - -func (a *appenderRecorder) AppendHistogram(ref storage.SeriesRef, ls labels.Labels, _ int64, _ *histogram.Histogram, _ *histogram.FloatHistogram) (storage.SeriesRef, error) { - a.records = append(a.records, appenderRecord{op: "AppendHistogram", ref: ref, ls: ls}) - if a.appendHistogramError != nil { - return 0, a.appendHistogramError - } - if ref == 0 { - ref = a.newRef() - } - a.setOutRef(ref) - return ref, nil -} - -func (a *appenderRecorder) AppendHistogramSTZeroSample(ref storage.SeriesRef, ls labels.Labels, _, _ int64, _ *histogram.Histogram, _ *histogram.FloatHistogram) (storage.SeriesRef, error) { - a.records = append(a.records, appenderRecord{op: "AppendHistogramSTZeroSample", ref: ref, ls: ls}) - if a.appendHistogramSTZeroSampleError != nil { - return 0, a.appendHistogramSTZeroSampleError - } - if ref == 0 { - ref = a.newRef() - } - a.setOutRef(ref) - return ref, nil -} - -func (a *appenderRecorder) UpdateMetadata(ref storage.SeriesRef, ls labels.Labels, _ metadata.Metadata) (storage.SeriesRef, error) { - a.records = append(a.records, appenderRecord{op: "UpdateMetadata", ref: ref, ls: ls}) - if a.updateMetadataError != nil { - return 0, a.updateMetadataError - } - a.setOutRef(ref) - return ref, nil -} - -func (a *appenderRecorder) AppendExemplar(ref storage.SeriesRef, ls labels.Labels, _ exemplar.Exemplar) (storage.SeriesRef, error) { - a.records = append(a.records, appenderRecord{op: "AppendExemplar", ref: ref, ls: ls}) - if a.appendExemplarError != nil { - return 0, a.appendExemplarError - } - a.setOutRef(ref) - return ref, nil -} - -func (a *appenderRecorder) Commit() error { - a.records = append(a.records, appenderRecord{op: "Commit"}) - return nil -} - -func (a *appenderRecorder) Rollback() error { - a.records = append(a.records, appenderRecord{op: "Rollback"}) - return nil -} - -func (*appenderRecorder) SetOptions(_ *storage.AppendOptions) { - panic("not implemented") -} - -func TestMetadataChangedLogic(t *testing.T) { - seriesLabels := labels.FromStrings(model.MetricNameLabel, "test_metric", "foo", "bar") - baseMetadata := Metadata{ - Metadata: metadata.Metadata{Type: model.MetricTypeCounter, Unit: "bytes", Help: "original"}, - MetricFamilyName: "test_metric", - } - - tests := []struct { - name string - appendMetadata bool - modifyMetadata func(Metadata) Metadata - expectWALCall bool - verifyCached func(*testing.T, metadata.Metadata) - }{ - { - name: "appendMetadata=false, no change", - appendMetadata: false, - modifyMetadata: func(m Metadata) Metadata { return m }, - expectWALCall: false, - verifyCached: func(t *testing.T, m metadata.Metadata) { require.Equal(t, "original", m.Help) }, - }, - { - name: "appendMetadata=false, help changes - cache updated, no WAL", - appendMetadata: false, - modifyMetadata: func(m Metadata) Metadata { m.Help = "changed"; return m }, - expectWALCall: false, - verifyCached: func(t *testing.T, m metadata.Metadata) { require.Equal(t, "changed", m.Help) }, - }, - { - name: "appendMetadata=true, help changes - cache and WAL updated", - appendMetadata: true, - modifyMetadata: func(m Metadata) Metadata { m.Help = "changed"; return m }, - expectWALCall: true, - verifyCached: func(t *testing.T, m metadata.Metadata) { require.Equal(t, "changed", m.Help) }, - }, - { - name: "appendMetadata=true, unit changes", - appendMetadata: true, - modifyMetadata: func(m Metadata) Metadata { m.Unit = "seconds"; return m }, - expectWALCall: true, - verifyCached: func(t *testing.T, m metadata.Metadata) { require.Equal(t, "seconds", m.Unit) }, - }, - { - name: "appendMetadata=true, type changes", - appendMetadata: true, - modifyMetadata: func(m Metadata) Metadata { m.Type = model.MetricTypeGauge; return m }, - expectWALCall: true, - verifyCached: func(t *testing.T, m metadata.Metadata) { require.Equal(t, model.MetricTypeGauge, m.Type) }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - app := &appenderRecorder{} - capp := NewCombinedAppender(app, promslog.NewNopLogger(), true, tt.appendMetadata, NewCombinedAppenderMetrics(prometheus.NewRegistry())) - - require.NoError(t, capp.AppendSample(seriesLabels.Copy(), baseMetadata, 1, 2, 42.0, nil)) - - modifiedMetadata := tt.modifyMetadata(baseMetadata) - app.records = nil - require.NoError(t, capp.AppendSample(seriesLabels.Copy(), modifiedMetadata, 1, 3, 43.0, nil)) - - hash := seriesLabels.Hash() - cached, exists := capp.(*combinedAppender).refs[hash] - require.True(t, exists) - tt.verifyCached(t, cached.meta) - - updateMetadataCalled := false - for _, record := range app.records { - if record.op == "UpdateMetadata" { - updateMetadataCalled = true - break - } - } - require.Equal(t, tt.expectWALCall, updateMetadataCalled) - }) - } -} diff --git a/storage/remote/otlptranslator/prometheusremotewrite/helper.go b/storage/remote/otlptranslator/prometheusremotewrite/helper.go index aa54433836..e670aa2f2d 100644 --- a/storage/remote/otlptranslator/prometheusremotewrite/helper.go +++ b/storage/remote/otlptranslator/prometheusremotewrite/helper.go @@ -30,6 +30,7 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/otlptranslator" + "github.com/prometheus/prometheus/storage" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" conventions "go.opentelemetry.io/collector/semconv/v1.6.1" @@ -64,8 +65,15 @@ const ( // Unpaired string values are ignored. String pairs overwrite OTLP labels if collisions happen and // if logOnOverwrite is true, the overwrite is logged. Resulting label names are sanitized. // If settings.PromoteResourceAttributes is not empty, it's a set of resource attributes that should be promoted to labels. -func (c *PrometheusConverter) createAttributes(resource pcommon.Resource, attributes pcommon.Map, scope scope, settings Settings, - ignoreAttrs []string, logOnOverwrite bool, meta Metadata, extras ...string, +func (c *PrometheusConverter) createAttributes( + resource pcommon.Resource, + attributes pcommon.Map, + scope scope, + settings Settings, + ignoreAttrs []string, + logOnOverwrite bool, + meta metadata.Metadata, + extras ...string, ) (labels.Labels, error) { resourceAttrs := resource.Attributes() serviceName, haveServiceName := resourceAttrs.Get(conventions.AttributeServiceName) @@ -222,8 +230,13 @@ func aggregationTemporality(metric pmetric.Metric) (pmetric.AggregationTemporali // with the user defined bucket boundaries of non-exponential OTel histograms. // However, work is under way to resolve this shortcoming through a feature called native histograms custom buckets: // https://github.com/prometheus/prometheus/issues/13485. -func (c *PrometheusConverter) addHistogramDataPoints(ctx context.Context, dataPoints pmetric.HistogramDataPointSlice, - resource pcommon.Resource, settings Settings, scope scope, meta Metadata, +func (c *PrometheusConverter) addHistogramDataPoints( + ctx context.Context, + dataPoints pmetric.HistogramDataPointSlice, + resource pcommon.Resource, + settings Settings, + scope scope, + appOpts storage.AOptions, ) error { for x := 0; x < dataPoints.Len(); x++ { if err := c.everyN.checkContext(ctx); err != nil { @@ -231,40 +244,35 @@ func (c *PrometheusConverter) addHistogramDataPoints(ctx context.Context, dataPo } pt := dataPoints.At(x) - timestamp := convertTimeStamp(pt.Timestamp()) - startTimestamp := convertTimeStamp(pt.StartTimestamp()) - baseLabels, err := c.createAttributes(resource, pt.Attributes(), scope, settings, nil, false, meta) + t := convertTimeStamp(pt.Timestamp()) + st := convertTimeStamp(pt.StartTimestamp()) + baseLabels, err := c.createAttributes(resource, pt.Attributes(), scope, settings, nil, false, appOpts.Metadata) if err != nil { return err } - baseName := meta.MetricFamilyName - // If the sum is unset, it indicates the _sum metric point should be // omitted if pt.HasSum() { - // treat sum as a sample in an individual TimeSeries val := pt.Sum() if pt.Flags().NoRecordedValue() { val = math.Float64frombits(value.StaleNaN) } - - sumlabels := c.addLabels(baseName+sumStr, baseLabels) - if err := c.appender.AppendSample(sumlabels, meta, startTimestamp, timestamp, val, nil); err != nil { + sumlabels := c.addLabels(appOpts.MetricFamilyName+sumStr, baseLabels) + if _, err := c.appender.Append(0, sumlabels, st, t, val, nil, nil, appOpts); err != nil { return err } } - // treat count as a sample in an individual TimeSeries val := float64(pt.Count()) if pt.Flags().NoRecordedValue() { val = math.Float64frombits(value.StaleNaN) } - - countlabels := c.addLabels(baseName+countStr, baseLabels) - if err := c.appender.AppendSample(countlabels, meta, startTimestamp, timestamp, val, nil); err != nil { + countlabels := c.addLabels(appOpts.MetricFamilyName+countStr, baseLabels) + if _, err = c.appender.Append(0, countlabels, st, t, val, nil, nil, appOpts); err != nil { return err } + exemplars, err := c.getPromExemplars(ctx, pt.Exemplars()) if err != nil { return err @@ -299,8 +307,9 @@ func (c *PrometheusConverter) addHistogramDataPoints(ctx context.Context, dataPo val = math.Float64frombits(value.StaleNaN) } boundStr := strconv.FormatFloat(bound, 'f', -1, 64) - labels := c.addLabels(baseName+bucketStr, baseLabels, leStr, boundStr) - if err := c.appender.AppendSample(labels, meta, startTimestamp, timestamp, val, currentBucketExemplars); err != nil { + bktLabels := c.addLabels(appOpts.MetricFamilyName+bucketStr, baseLabels, leStr, boundStr) + appOpts.Exemplars = currentBucketExemplars + if _, err = c.appender.Append(0, bktLabels, st, t, val, nil, nil, appOpts); err != nil { return err } } @@ -309,12 +318,12 @@ func (c *PrometheusConverter) addHistogramDataPoints(ctx context.Context, dataPo if pt.Flags().NoRecordedValue() { val = math.Float64frombits(value.StaleNaN) } - infLabels := c.addLabels(baseName+bucketStr, baseLabels, leStr, pInfStr) - if err := c.appender.AppendSample(infLabels, meta, startTimestamp, timestamp, val, exemplars[nextExemplarIdx:]); err != nil { + infLabels := c.addLabels(appOpts.MetricFamilyName+bucketStr, baseLabels, leStr, pInfStr) + appOpts.Exemplars = exemplars[nextExemplarIdx:] + if _, err = c.appender.Append(0, infLabels, st, t, val, nil, nil, appOpts); err != nil { return err } } - return nil } @@ -424,8 +433,13 @@ func findMinAndMaxTimestamps(metric pmetric.Metric, minTimestamp, maxTimestamp p return minTimestamp, maxTimestamp } -func (c *PrometheusConverter) addSummaryDataPoints(ctx context.Context, dataPoints pmetric.SummaryDataPointSlice, resource pcommon.Resource, - settings Settings, scope scope, meta Metadata, +func (c *PrometheusConverter) addSummaryDataPoints( + ctx context.Context, + dataPoints pmetric.SummaryDataPointSlice, + resource pcommon.Resource, + settings Settings, + scope scope, + appOpts storage.AOptions, ) error { for x := 0; x < dataPoints.Len(); x++ { if err := c.everyN.checkContext(ctx); err != nil { @@ -433,33 +447,28 @@ func (c *PrometheusConverter) addSummaryDataPoints(ctx context.Context, dataPoin } pt := dataPoints.At(x) - timestamp := convertTimeStamp(pt.Timestamp()) - startTimestamp := convertTimeStamp(pt.StartTimestamp()) - baseLabels, err := c.createAttributes(resource, pt.Attributes(), scope, settings, nil, false, meta) + t := convertTimeStamp(pt.Timestamp()) + st := convertTimeStamp(pt.StartTimestamp()) + baseLabels, err := c.createAttributes(resource, pt.Attributes(), scope, settings, nil, false, appOpts.Metadata) if err != nil { return err } - baseName := meta.MetricFamilyName - - // treat sum as a sample in an individual TimeSeries val := pt.Sum() if pt.Flags().NoRecordedValue() { val = math.Float64frombits(value.StaleNaN) } - // sum and count of the summary should append suffix to baseName - sumlabels := c.addLabels(baseName+sumStr, baseLabels) - if err := c.appender.AppendSample(sumlabels, meta, startTimestamp, timestamp, val, nil); err != nil { + sumlabels := c.addLabels(appOpts.MetricFamilyName+sumStr, baseLabels) + if _, err = c.appender.Append(0, sumlabels, st, t, val, nil, nil, appOpts); err != nil { return err } - // treat count as a sample in an individual TimeSeries val = float64(pt.Count()) if pt.Flags().NoRecordedValue() { val = math.Float64frombits(value.StaleNaN) } - countlabels := c.addLabels(baseName+countStr, baseLabels) - if err := c.appender.AppendSample(countlabels, meta, startTimestamp, timestamp, val, nil); err != nil { + countlabels := c.addLabels(appOpts.MetricFamilyName+countStr, baseLabels) + if _, err = c.appender.Append(0, countlabels, st, t, val, nil, nil, appOpts); err != nil { return err } @@ -471,13 +480,12 @@ func (c *PrometheusConverter) addSummaryDataPoints(ctx context.Context, dataPoin val = math.Float64frombits(value.StaleNaN) } percentileStr := strconv.FormatFloat(qt.Quantile(), 'f', -1, 64) - qtlabels := c.addLabels(baseName, baseLabels, quantileStr, percentileStr) - if err := c.appender.AppendSample(qtlabels, meta, startTimestamp, timestamp, val, nil); err != nil { + qtlabels := c.addLabels(appOpts.MetricFamilyName, baseLabels, quantileStr, percentileStr) + if _, err = c.appender.Append(0, qtlabels, st, t, val, nil, nil, appOpts); err != nil { return err } } } - return nil } @@ -530,7 +538,7 @@ func (c *PrometheusConverter) addResourceTargetInfo(resource pcommon.Resource, s // Do not pass identifying attributes as ignoreAttrs below. identifyingAttrs = nil } - meta := Metadata{ + appOpts := storage.AOptions{ Metadata: metadata.Metadata{ Type: model.MetricTypeGauge, Help: "Target metadata", @@ -538,7 +546,7 @@ func (c *PrometheusConverter) addResourceTargetInfo(resource pcommon.Resource, s MetricFamilyName: name, } // TODO: should target info have the __type__ metadata label? - lbls, err := c.createAttributes(resource, attributes, scope{}, settings, identifyingAttrs, false, Metadata{}, model.MetricNameLabel, name) + lbls, err := c.createAttributes(resource, attributes, scope{}, settings, identifyingAttrs, false, metadata.Metadata{}, model.MetricNameLabel, name) if err != nil { return err } @@ -569,10 +577,10 @@ func (c *PrometheusConverter) addResourceTargetInfo(resource pcommon.Resource, s var key targetInfoKey for timestamp := earliestTimestamp; timestamp.Before(latestTimestamp); timestamp = timestamp.Add(interval) { - timestampMs := timestamp.UnixMilli() + t := timestamp.UnixMilli() key = targetInfoKey{ labelsHash: labelsHash, - timestamp: timestampMs, + timestamp: t, } if _, exists := c.seenTargetInfo[key]; exists { // Skip duplicate. @@ -580,23 +588,25 @@ func (c *PrometheusConverter) addResourceTargetInfo(resource pcommon.Resource, s } c.seenTargetInfo[key] = struct{}{} - if err := c.appender.AppendSample(lbls, meta, 0, timestampMs, float64(1), nil); err != nil { + _, err = c.appender.Append(0, lbls, 0, t, 1.0, nil, nil, appOpts) + if err != nil { return err } } // Append the final sample at latestTimestamp. - finalTimestampMs := latestTimestamp.UnixMilli() + finalT := latestTimestamp.UnixMilli() key = targetInfoKey{ labelsHash: labelsHash, - timestamp: finalTimestampMs, + timestamp: finalT, } if _, exists := c.seenTargetInfo[key]; exists { return nil } c.seenTargetInfo[key] = struct{}{} - return c.appender.AppendSample(lbls, meta, 0, finalTimestampMs, float64(1), nil) + _, err = c.appender.Append(0, lbls, 0, finalT, 1.0, nil, nil, appOpts) + return err } // convertTimeStamp converts OTLP timestamp in ns to timestamp in ms. diff --git a/storage/remote/otlptranslator/prometheusremotewrite/helper_test.go b/storage/remote/otlptranslator/prometheusremotewrite/helper_test.go index 893fe97ec4..41529fa8b6 100644 --- a/storage/remote/otlptranslator/prometheusremotewrite/helper_test.go +++ b/storage/remote/otlptranslator/prometheusremotewrite/helper_test.go @@ -18,6 +18,7 @@ package prometheusremotewrite import ( "context" + "errors" "slices" "strings" "testing" @@ -25,16 +26,54 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/otlptranslator" + "github.com/prometheus/prometheus/storage" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/model/metadata" "github.com/prometheus/prometheus/prompb" "github.com/prometheus/prometheus/util/testutil" ) +type statsAppender struct { + samples int + histograms int + metadata int +} + +func (a *statsAppender) Append(ref storage.SeriesRef, ls labels.Labels, st, t int64, v float64, h *histogram.Histogram, fh *histogram.FloatHistogram, opts storage.AOptions) (storage.SeriesRef, error) { + if fh != nil { + return 0, errors.New("mockAppender.Append: mock appender is not nil") + } + if h != nil { + a.histograms++ + } else { + a.samples++ + } + + if !opts.Metadata.IsEmpty() { + a.metadata++ + } + + if ref == 0 { + // Use labels hash as a stand-in for unique series reference, to avoid having to track all series. + ref = storage.SeriesRef(ls.Hash()) + } + return ref, nil +} + +func (a *statsAppender) Commit() error { + return nil +} + +func (a *statsAppender) Rollback() error { + return nil +} + func TestCreateAttributes(t *testing.T) { resourceAttrs := map[string]string{ "service.name": "service name", @@ -389,7 +428,7 @@ func TestCreateAttributes(t *testing.T) { } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - c := NewPrometheusConverter(&mockCombinedAppender{}) + c := NewPrometheusConverter(&mockAppender{}) settings := Settings{ PromoteResourceAttributes: NewPromoteResourceAttributes(config.OTLPConfig{ PromoteAllResourceAttributes: tc.promoteAllResourceAttributes, @@ -413,7 +452,7 @@ func TestCreateAttributes(t *testing.T) { if tc.attrs != (pcommon.Map{}) { testAttrs = tc.attrs } - lbls, err := c.createAttributes(testResource, testAttrs, tc.scope, settings, tc.ignoreAttrs, false, Metadata{}, model.MetricNameLabel, "test_metric") + lbls, err := c.createAttributes(testResource, testAttrs, tc.scope, settings, tc.ignoreAttrs, false, metadata.Metadata{}, model.MetricNameLabel, "test_metric") require.NoError(t, err) testutil.RequireEqual(t, tc.expectedLabels, lbls) @@ -641,10 +680,10 @@ func TestPrometheusConverter_AddSummaryDataPoints(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { metric := tt.metric() - mockAppender := &mockCombinedAppender{} - converter := NewPrometheusConverter(mockAppender) + mApp := &mockAppender{} + converter := NewPrometheusConverter(mApp) - converter.addSummaryDataPoints( + require.NoError(t, converter.addSummaryDataPoints( context.Background(), metric.Summary().DataPoints(), pcommon.NewResource(), @@ -652,13 +691,13 @@ func TestPrometheusConverter_AddSummaryDataPoints(t *testing.T) { PromoteScopeMetadata: tt.promoteScope, }, tt.scope, - Metadata{ + storage.AOptions{ MetricFamilyName: metric.Name(), }, - ) - require.NoError(t, mockAppender.Commit()) + )) + require.NoError(t, mApp.Commit()) - requireEqual(t, tt.want(), mockAppender.samples) + requireEqual(t, tt.want(), mApp.samples) }) } } @@ -804,10 +843,10 @@ func TestPrometheusConverter_AddHistogramDataPoints(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { metric := tt.metric() - mockAppender := &mockCombinedAppender{} - converter := NewPrometheusConverter(mockAppender) + mApp := &mockAppender{} + converter := NewPrometheusConverter(mApp) - converter.addHistogramDataPoints( + require.NoError(t, converter.addHistogramDataPoints( context.Background(), metric.Histogram().DataPoints(), pcommon.NewResource(), @@ -815,20 +854,20 @@ func TestPrometheusConverter_AddHistogramDataPoints(t *testing.T) { PromoteScopeMetadata: tt.promoteScope, }, tt.scope, - Metadata{ + storage.AOptions{ MetricFamilyName: metric.Name(), }, - ) - require.NoError(t, mockAppender.Commit()) + )) + require.NoError(t, mApp.Commit()) - requireEqual(t, tt.want(), mockAppender.samples) + requireEqual(t, tt.want(), mApp.samples) }) } } func TestGetPromExemplars(t *testing.T) { ctx := context.Background() - c := NewPrometheusConverter(&mockCombinedAppender{}) + c := NewPrometheusConverter(&mockAppender{}) t.Run("Exemplars with int value", func(t *testing.T) { es := pmetric.NewExemplarSlice() diff --git a/storage/remote/otlptranslator/prometheusremotewrite/histograms.go b/storage/remote/otlptranslator/prometheusremotewrite/histograms.go index c93a00db76..c4395fdc9a 100644 --- a/storage/remote/otlptranslator/prometheusremotewrite/histograms.go +++ b/storage/remote/otlptranslator/prometheusremotewrite/histograms.go @@ -22,6 +22,7 @@ import ( "math" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/storage" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" @@ -34,9 +35,14 @@ const defaultZeroThreshold = 1e-128 // addExponentialHistogramDataPoints adds OTel exponential histogram data points to the corresponding time series // as native histogram samples. -func (c *PrometheusConverter) addExponentialHistogramDataPoints(ctx context.Context, dataPoints pmetric.ExponentialHistogramDataPointSlice, - resource pcommon.Resource, settings Settings, temporality pmetric.AggregationTemporality, - scope scope, meta Metadata, +func (c *PrometheusConverter) addExponentialHistogramDataPoints( + ctx context.Context, + dataPoints pmetric.ExponentialHistogramDataPointSlice, + resource pcommon.Resource, + settings Settings, + temporality pmetric.AggregationTemporality, + scope scope, + appOpts storage.AOptions, ) (annotations.Annotations, error) { var annots annotations.Annotations for x := 0; x < dataPoints.Len(); x++ { @@ -59,21 +65,23 @@ func (c *PrometheusConverter) addExponentialHistogramDataPoints(ctx context.Cont settings, nil, true, - meta, + appOpts.Metadata, model.MetricNameLabel, - meta.MetricFamilyName, + appOpts.MetricFamilyName, ) if err != nil { return annots, err } - ts := convertTimeStamp(pt.Timestamp()) + t := convertTimeStamp(pt.Timestamp()) st := convertTimeStamp(pt.StartTimestamp()) exemplars, err := c.getPromExemplars(ctx, pt.Exemplars()) if err != nil { return annots, err } - // OTel exponential histograms are always Int Histograms. - if err = c.appender.AppendHistogram(lbls, meta, st, ts, hp, exemplars); err != nil { + appOpts.Exemplars = exemplars + + // OTel exponential histograms are always integer histograms. + if _, err = c.appender.Append(0, lbls, st, t, 0, hp, nil, appOpts); err != nil { return annots, err } } @@ -252,9 +260,14 @@ func convertBucketsLayout(bucketCounts []uint64, offset, scaleDown int32, adjust return spans, deltas } -func (c *PrometheusConverter) addCustomBucketsHistogramDataPoints(ctx context.Context, dataPoints pmetric.HistogramDataPointSlice, - resource pcommon.Resource, settings Settings, temporality pmetric.AggregationTemporality, - scope scope, meta Metadata, +func (c *PrometheusConverter) addCustomBucketsHistogramDataPoints( + ctx context.Context, + dataPoints pmetric.HistogramDataPointSlice, + resource pcommon.Resource, + settings Settings, + temporality pmetric.AggregationTemporality, + scope scope, + appOpts storage.AOptions, ) (annotations.Annotations, error) { var annots annotations.Annotations @@ -278,20 +291,21 @@ func (c *PrometheusConverter) addCustomBucketsHistogramDataPoints(ctx context.Co settings, nil, true, - meta, + appOpts.Metadata, model.MetricNameLabel, - meta.MetricFamilyName, + appOpts.MetricFamilyName, ) if err != nil { return annots, err } - ts := convertTimeStamp(pt.Timestamp()) + t := convertTimeStamp(pt.Timestamp()) st := convertTimeStamp(pt.StartTimestamp()) exemplars, err := c.getPromExemplars(ctx, pt.Exemplars()) if err != nil { return annots, err } - if err = c.appender.AppendHistogram(lbls, meta, st, ts, hp, exemplars); err != nil { + appOpts.Exemplars = exemplars + if _, err = c.appender.Append(0, lbls, st, t, 0, hp, nil, appOpts); err != nil { return annots, err } } diff --git a/storage/remote/otlptranslator/prometheusremotewrite/histograms_test.go b/storage/remote/otlptranslator/prometheusremotewrite/histograms_test.go index 22e654ab9c..a7d40e5a4e 100644 --- a/storage/remote/otlptranslator/prometheusremotewrite/histograms_test.go +++ b/storage/remote/otlptranslator/prometheusremotewrite/histograms_test.go @@ -24,6 +24,7 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/otlptranslator" + "github.com/prometheus/prometheus/storage" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" @@ -854,8 +855,8 @@ func TestPrometheusConverter_addExponentialHistogramDataPoints(t *testing.T) { t.Run(tt.name, func(t *testing.T) { metric := tt.metric() - mockAppender := &mockCombinedAppender{} - converter := NewPrometheusConverter(mockAppender) + mApp := &mockAppender{} + converter := NewPrometheusConverter(mApp) namer := otlptranslator.MetricNamer{ WithMetricSuffixes: true, } @@ -870,16 +871,16 @@ func TestPrometheusConverter_addExponentialHistogramDataPoints(t *testing.T) { }, pmetric.AggregationTemporalityCumulative, tt.scope, - Metadata{ + storage.AOptions{ MetricFamilyName: name, }, ) require.NoError(t, err) require.Empty(t, annots) - require.NoError(t, mockAppender.Commit()) + require.NoError(t, mApp.Commit()) - requireEqual(t, tt.wantSeries(), mockAppender.histograms) + requireEqual(t, tt.wantSeries(), mApp.histograms) }) } } @@ -1327,8 +1328,8 @@ func TestPrometheusConverter_addCustomBucketsHistogramDataPoints(t *testing.T) { t.Run(tt.name, func(t *testing.T) { metric := tt.metric() - mockAppender := &mockCombinedAppender{} - converter := NewPrometheusConverter(mockAppender) + mApp := &mockAppender{} + converter := NewPrometheusConverter(mApp) namer := otlptranslator.MetricNamer{ WithMetricSuffixes: true, } @@ -1344,7 +1345,7 @@ func TestPrometheusConverter_addCustomBucketsHistogramDataPoints(t *testing.T) { }, pmetric.AggregationTemporalityCumulative, tt.scope, - Metadata{ + storage.AOptions{ MetricFamilyName: name, }, ) @@ -1352,9 +1353,9 @@ func TestPrometheusConverter_addCustomBucketsHistogramDataPoints(t *testing.T) { require.NoError(t, err) require.Empty(t, annots) - require.NoError(t, mockAppender.Commit()) + require.NoError(t, mApp.Commit()) - requireEqual(t, tt.wantSeries(), mockAppender.histograms) + requireEqual(t, tt.wantSeries(), mApp.histograms) }) } } diff --git a/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go b/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go index f43e4964b1..f1cf84df6f 100644 --- a/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go +++ b/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go @@ -24,6 +24,7 @@ import ( "time" "github.com/prometheus/otlptranslator" + "github.com/prometheus/prometheus/storage" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" "go.uber.org/multierr" @@ -67,7 +68,7 @@ type PrometheusConverter struct { everyN everyNTimes scratchBuilder labels.ScratchBuilder builder *labels.Builder - appender CombinedAppender + appender storage.AppenderV2 // seenTargetInfo tracks target_info samples within a batch to prevent duplicates. seenTargetInfo map[targetInfoKey]struct{} } @@ -78,7 +79,7 @@ type targetInfoKey struct { timestamp int64 } -func NewPrometheusConverter(appender CombinedAppender) *PrometheusConverter { +func NewPrometheusConverter(appender storage.AppenderV2) *PrometheusConverter { return &PrometheusConverter{ scratchBuilder: labels.NewScratchBuilder(0), builder: labels.NewBuilder(labels.EmptyLabels()), @@ -128,7 +129,7 @@ func newScopeFromScopeMetrics(scopeMetrics pmetric.ScopeMetrics) scope { } } -// FromMetrics converts pmetric.Metrics to Prometheus remote write format. +// FromMetrics appends pmetric.Metrics to storage.AppenderV2. func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metrics, settings Settings) (annots annotations.Annotations, errs error) { namer := otlptranslator.MetricNamer{ Namespace: settings.Namespace, @@ -184,7 +185,8 @@ func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metric errs = multierr.Append(errs, err) continue } - meta := Metadata{ + + appOpts := storage.AOptions{ Metadata: metadata.Metadata{ Type: otelMetricTypeToPromMetricType(metric), Unit: unitNamer.Build(metric.Unit()), @@ -202,7 +204,7 @@ func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metric errs = multierr.Append(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name())) break } - if err := c.addGaugeNumberDataPoints(ctx, dataPoints, resource, settings, scope, meta); err != nil { + if err := c.addGaugeNumberDataPoints(ctx, dataPoints, resource, settings, scope, appOpts); err != nil { errs = multierr.Append(errs, err) if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { return annots, errs @@ -214,7 +216,7 @@ func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metric errs = multierr.Append(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name())) break } - if err := c.addSumNumberDataPoints(ctx, dataPoints, resource, settings, scope, meta); err != nil { + if err := c.addSumNumberDataPoints(ctx, dataPoints, resource, settings, scope, appOpts); err != nil { errs = multierr.Append(errs, err) if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { return annots, errs @@ -228,7 +230,7 @@ func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metric } if settings.ConvertHistogramsToNHCB { ws, err := c.addCustomBucketsHistogramDataPoints( - ctx, dataPoints, resource, settings, temporality, scope, meta, + ctx, dataPoints, resource, settings, temporality, scope, appOpts, ) annots.Merge(ws) if err != nil { @@ -238,7 +240,7 @@ func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metric } } } else { - if err := c.addHistogramDataPoints(ctx, dataPoints, resource, settings, scope, meta); err != nil { + if err := c.addHistogramDataPoints(ctx, dataPoints, resource, settings, scope, appOpts); err != nil { errs = multierr.Append(errs, err) if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { return annots, errs @@ -258,7 +260,7 @@ func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metric settings, temporality, scope, - meta, + appOpts, ) annots.Merge(ws) if err != nil { @@ -273,7 +275,7 @@ func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metric errs = multierr.Append(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name())) break } - if err := c.addSummaryDataPoints(ctx, dataPoints, resource, settings, scope, meta); err != nil { + if err := c.addSummaryDataPoints(ctx, dataPoints, resource, settings, scope, appOpts); err != nil { errs = multierr.Append(errs, err) if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { return annots, errs diff --git a/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw_test.go b/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw_test.go index e409b4e8b5..859ce0e8bc 100644 --- a/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw_test.go +++ b/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw_test.go @@ -22,20 +22,16 @@ import ( "testing" "time" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" - "github.com/prometheus/common/promslog" "github.com/prometheus/otlptranslator" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" "go.opentelemetry.io/collector/pdata/pmetric/pmetricotlp" - "github.com/prometheus/prometheus/model/exemplar" "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/metadata" - "github.com/prometheus/prometheus/storage" ) func TestFromMetrics(t *testing.T) { @@ -81,7 +77,7 @@ func TestFromMetrics(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - mockAppender := &mockCombinedAppender{} + mockAppender := &mockAppender{} converter := NewPrometheusConverter(mockAppender) payload, wantPromMetrics := createExportRequest(5, 128, 128, 2, 0, tc.settings, tc.temporality) seenFamilyNames := map[string]struct{}{} @@ -153,7 +149,7 @@ func TestFromMetrics(t *testing.T) { generateAttributes(h.Attributes(), "series", 1) - mockAppender := &mockCombinedAppender{} + mockAppender := &mockAppender{} converter := NewPrometheusConverter(mockAppender) annots, err := converter.FromMetrics( context.Background(), @@ -176,7 +172,7 @@ func TestFromMetrics(t *testing.T) { t.Run("context cancellation", func(t *testing.T) { settings := Settings{} - converter := NewPrometheusConverter(&mockCombinedAppender{}) + converter := NewPrometheusConverter(&mockAppender{}) ctx, cancel := context.WithCancel(context.Background()) // Verify that converter.FromMetrics respects cancellation. cancel() @@ -189,7 +185,7 @@ func TestFromMetrics(t *testing.T) { t.Run("context timeout", func(t *testing.T) { settings := Settings{} - converter := NewPrometheusConverter(&mockCombinedAppender{}) + converter := NewPrometheusConverter(&mockAppender{}) // Verify that converter.FromMetrics respects timeout. ctx, cancel := context.WithTimeout(context.Background(), 0) t.Cleanup(cancel) @@ -222,7 +218,7 @@ func TestFromMetrics(t *testing.T) { generateAttributes(h.Attributes(), "series", 10) } - converter := NewPrometheusConverter(&mockCombinedAppender{}) + converter := NewPrometheusConverter(&mockAppender{}) annots, err := converter.FromMetrics(context.Background(), request.Metrics(), Settings{}) require.NoError(t, err) require.NotEmpty(t, annots) @@ -255,7 +251,7 @@ func TestFromMetrics(t *testing.T) { generateAttributes(h.Attributes(), "series", 10) } - converter := NewPrometheusConverter(&mockCombinedAppender{}) + converter := NewPrometheusConverter(&mockAppender{}) annots, err := converter.FromMetrics( context.Background(), request.Metrics(), @@ -303,7 +299,7 @@ func TestFromMetrics(t *testing.T) { } } - mockAppender := &mockCombinedAppender{} + mockAppender := &mockAppender{} converter := NewPrometheusConverter(mockAppender) annots, err := converter.FromMetrics( context.Background(), @@ -403,7 +399,7 @@ func TestFromMetrics(t *testing.T) { generateAttributes(point2.Attributes(), "series", 1) } - mockAppender := &mockCombinedAppender{} + mockAppender := &mockAppender{} converter := NewPrometheusConverter(mockAppender) annots, err := converter.FromMetrics( context.Background(), @@ -660,7 +656,7 @@ func TestTemporality(t *testing.T) { s.CopyTo(sm.Metrics().AppendEmpty()) } - mockAppender := &mockCombinedAppender{} + mockAppender := &mockAppender{} c := NewPrometheusConverter(mockAppender) settings := Settings{ AllowDeltaTemporality: tc.allowDelta, @@ -1061,14 +1057,11 @@ func BenchmarkPrometheusConverter_FromMetrics(b *testing.B) { settings, pmetric.AggregationTemporalityCumulative, ) - appMetrics := NewCombinedAppenderMetrics(prometheus.NewRegistry()) - noOpLogger := promslog.NewNopLogger() b.ResetTimer() for b.Loop() { - app := &noOpAppender{} - mockAppender := NewCombinedAppender(app, noOpLogger, false, false, appMetrics) - converter := NewPrometheusConverter(mockAppender) + app := &statsAppender{} + converter := NewPrometheusConverter(app) annots, err := converter.FromMetrics(context.Background(), payload.Metrics(), settings) require.NoError(b, err) require.Empty(b, annots) @@ -1092,53 +1085,6 @@ func BenchmarkPrometheusConverter_FromMetrics(b *testing.B) { } } -type noOpAppender struct { - samples int - histograms int - metadata int -} - -var _ storage.Appender = &noOpAppender{} - -func (a *noOpAppender) Append(_ storage.SeriesRef, _ labels.Labels, _ int64, _ float64) (storage.SeriesRef, error) { - a.samples++ - return 1, nil -} - -func (*noOpAppender) AppendSTZeroSample(_ storage.SeriesRef, _ labels.Labels, _, _ int64) (storage.SeriesRef, error) { - return 1, nil -} - -func (a *noOpAppender) AppendHistogram(_ storage.SeriesRef, _ labels.Labels, _ int64, _ *histogram.Histogram, _ *histogram.FloatHistogram) (storage.SeriesRef, error) { - a.histograms++ - return 1, nil -} - -func (*noOpAppender) AppendHistogramSTZeroSample(_ storage.SeriesRef, _ labels.Labels, _, _ int64, _ *histogram.Histogram, _ *histogram.FloatHistogram) (storage.SeriesRef, error) { - return 1, nil -} - -func (a *noOpAppender) UpdateMetadata(_ storage.SeriesRef, _ labels.Labels, _ metadata.Metadata) (storage.SeriesRef, error) { - a.metadata++ - return 1, nil -} - -func (*noOpAppender) AppendExemplar(_ storage.SeriesRef, _ labels.Labels, _ exemplar.Exemplar) (storage.SeriesRef, error) { - return 1, nil -} - -func (*noOpAppender) Commit() error { - return nil -} - -func (*noOpAppender) Rollback() error { - return nil -} - -func (*noOpAppender) SetOptions(_ *storage.AppendOptions) { - panic("not implemented") -} - type wantPrometheusMetric struct { name string familyName string diff --git a/storage/remote/otlptranslator/prometheusremotewrite/number_data_points.go b/storage/remote/otlptranslator/prometheusremotewrite/number_data_points.go index 8f30dbb6b6..2b8270e886 100644 --- a/storage/remote/otlptranslator/prometheusremotewrite/number_data_points.go +++ b/storage/remote/otlptranslator/prometheusremotewrite/number_data_points.go @@ -21,14 +21,20 @@ import ( "math" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/storage" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" "github.com/prometheus/prometheus/model/value" ) -func (c *PrometheusConverter) addGaugeNumberDataPoints(ctx context.Context, dataPoints pmetric.NumberDataPointSlice, - resource pcommon.Resource, settings Settings, scope scope, meta Metadata, +func (c *PrometheusConverter) addGaugeNumberDataPoints( + ctx context.Context, + dataPoints pmetric.NumberDataPointSlice, + resource pcommon.Resource, + settings Settings, + scope scope, + appOpts storage.AOptions, ) error { for x := 0; x < dataPoints.Len(); x++ { if err := c.everyN.checkContext(ctx); err != nil { @@ -43,13 +49,14 @@ func (c *PrometheusConverter) addGaugeNumberDataPoints(ctx context.Context, data settings, nil, true, - meta, + appOpts.Metadata, model.MetricNameLabel, - meta.MetricFamilyName, + appOpts.MetricFamilyName, ) if err != nil { return err } + var val float64 switch pt.ValueType() { case pmetric.NumberDataPointValueTypeInt: @@ -57,21 +64,26 @@ func (c *PrometheusConverter) addGaugeNumberDataPoints(ctx context.Context, data case pmetric.NumberDataPointValueTypeDouble: val = pt.DoubleValue() } + if pt.Flags().NoRecordedValue() { val = math.Float64frombits(value.StaleNaN) } - ts := convertTimeStamp(pt.Timestamp()) + t := convertTimeStamp(pt.Timestamp()) st := convertTimeStamp(pt.StartTimestamp()) - if err := c.appender.AppendSample(labels, meta, st, ts, val, nil); err != nil { + if _, err = c.appender.Append(0, labels, st, t, val, nil, nil, appOpts); err != nil { return err } } - return nil } -func (c *PrometheusConverter) addSumNumberDataPoints(ctx context.Context, dataPoints pmetric.NumberDataPointSlice, - resource pcommon.Resource, settings Settings, scope scope, meta Metadata, +func (c *PrometheusConverter) addSumNumberDataPoints( + ctx context.Context, + dataPoints pmetric.NumberDataPointSlice, + resource pcommon.Resource, + settings Settings, + scope scope, + appOpts storage.AOptions, ) error { for x := 0; x < dataPoints.Len(); x++ { if err := c.everyN.checkContext(ctx); err != nil { @@ -79,6 +91,7 @@ func (c *PrometheusConverter) addSumNumberDataPoints(ctx context.Context, dataPo } pt := dataPoints.At(x) + lbls, err := c.createAttributes( resource, pt.Attributes(), @@ -86,12 +99,12 @@ func (c *PrometheusConverter) addSumNumberDataPoints(ctx context.Context, dataPo settings, nil, true, - meta, + appOpts.Metadata, model.MetricNameLabel, - meta.MetricFamilyName, + appOpts.MetricFamilyName, ) if err != nil { - return nil + return err // NOTE: Previously it was nil, was it a bug? } var val float64 switch pt.ValueType() { @@ -100,16 +113,19 @@ func (c *PrometheusConverter) addSumNumberDataPoints(ctx context.Context, dataPo case pmetric.NumberDataPointValueTypeDouble: val = pt.DoubleValue() } + if pt.Flags().NoRecordedValue() { val = math.Float64frombits(value.StaleNaN) } - ts := convertTimeStamp(pt.Timestamp()) + t := convertTimeStamp(pt.Timestamp()) st := convertTimeStamp(pt.StartTimestamp()) exemplars, err := c.getPromExemplars(ctx, pt.Exemplars()) if err != nil { return err } - if err := c.appender.AppendSample(lbls, meta, st, ts, val, exemplars); err != nil { + + appOpts.Exemplars = exemplars + if _, err = c.appender.Append(0, lbls, st, t, val, nil, nil, appOpts); err != nil { return err } } diff --git a/storage/remote/otlptranslator/prometheusremotewrite/number_data_points_test.go b/storage/remote/otlptranslator/prometheusremotewrite/number_data_points_test.go index 32435020c5..7ce90d13f6 100644 --- a/storage/remote/otlptranslator/prometheusremotewrite/number_data_points_test.go +++ b/storage/remote/otlptranslator/prometheusremotewrite/number_data_points_test.go @@ -22,6 +22,7 @@ import ( "time" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/storage" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" @@ -112,10 +113,10 @@ func TestPrometheusConverter_addGaugeNumberDataPoints(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { metric := tt.metric() - mockAppender := &mockCombinedAppender{} - converter := NewPrometheusConverter(mockAppender) + mApp := &mockAppender{} + converter := NewPrometheusConverter(mApp) - converter.addGaugeNumberDataPoints( + require.NoError(t, converter.addGaugeNumberDataPoints( context.Background(), metric.Gauge().DataPoints(), pcommon.NewResource(), @@ -123,13 +124,13 @@ func TestPrometheusConverter_addGaugeNumberDataPoints(t *testing.T) { PromoteScopeMetadata: tt.promoteScope, }, tt.scope, - Metadata{ + storage.AOptions{ MetricFamilyName: metric.Name(), }, - ) - require.NoError(t, mockAppender.Commit()) + )) + require.NoError(t, mApp.Commit()) - requireEqual(t, tt.want(), mockAppender.samples) + requireEqual(t, tt.want(), mApp.samples) }) } } @@ -342,10 +343,10 @@ func TestPrometheusConverter_addSumNumberDataPoints(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { metric := tt.metric() - mockAppender := &mockCombinedAppender{} + mockAppender := &mockAppender{} converter := NewPrometheusConverter(mockAppender) - converter.addSumNumberDataPoints( + require.NoError(t, converter.addSumNumberDataPoints( context.Background(), metric.Sum().DataPoints(), pcommon.NewResource(), @@ -353,10 +354,10 @@ func TestPrometheusConverter_addSumNumberDataPoints(t *testing.T) { PromoteScopeMetadata: tt.promoteScope, }, tt.scope, - Metadata{ + storage.AOptions{ MetricFamilyName: metric.Name(), }, - ) + )) require.NoError(t, mockAppender.Commit()) requireEqual(t, tt.want(), mockAppender.samples) diff --git a/storage/remote/write_handler.go b/storage/remote/write_handler.go index b95c85b6c4..606c42ea56 100644 --- a/storage/remote/write_handler.go +++ b/storage/remote/write_handler.go @@ -28,6 +28,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/exemplar" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/consumer" "go.opentelemetry.io/collector/pdata/pmetric" @@ -35,7 +36,6 @@ import ( "go.opentelemetry.io/otel/metric/noop" "github.com/prometheus/prometheus/config" - "github.com/prometheus/prometheus/model/exemplar" "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/timestamp" @@ -48,14 +48,12 @@ import ( type writeHandler struct { logger *slog.Logger - appendable storage.Appendable + appendable storage.AppendableV2 samplesWithInvalidLabelsTotal prometheus.Counter samplesAppendedWithoutMetadata prometheus.Counter - ingestSTZeroSample bool enableTypeAndUnitLabels bool - appendMetadata bool } const maxAheadTime = 10 * time.Minute @@ -65,7 +63,7 @@ const maxAheadTime = 10 * time.Minute // // NOTE(bwplotka): When accepting v2 proto and spec, partial writes are possible // as per https://prometheus.io/docs/specs/remote_write_spec_2_0/#partial-write. -func NewWriteHandler(logger *slog.Logger, reg prometheus.Registerer, appendable storage.Appendable, acceptedMsgs remoteapi.MessageTypes, ingestSTZeroSample, enableTypeAndUnitLabels, appendMetadata bool) http.Handler { +func NewWriteHandler(logger *slog.Logger, reg prometheus.Registerer, appendable storage.AppendableV2, acceptedMsgs remoteapi.MessageTypes, enableTypeAndUnitLabels bool) http.Handler { h := &writeHandler{ logger: logger, appendable: appendable, @@ -82,9 +80,7 @@ func NewWriteHandler(logger *slog.Logger, reg prometheus.Registerer, appendable Help: "The total number of received remote write samples (and histogram samples) which were ingested without corresponding metadata.", }), - ingestSTZeroSample: ingestSTZeroSample, enableTypeAndUnitLabels: enableTypeAndUnitLabels, - appendMetadata: appendMetadata, } return remoteapi.NewWriteHandler(h, acceptedMsgs, remoteapi.WithWriteHandlerLogger(logger)) } @@ -155,9 +151,9 @@ func (h *writeHandler) write(ctx context.Context, req *prompb.WriteRequest) (err samplesWithInvalidLabels := 0 samplesAppended := 0 - app := &remoteWriteAppender{ - Appender: h.appendable.Appender(ctx), - maxTime: timestamp.FromTime(time.Now().Add(maxAheadTime)), + app := &validationAppender{ + AppenderV2: h.appendable.AppenderV2(ctx), + maxTime: timestamp.FromTime(time.Now().Add(maxAheadTime)), } defer func() { @@ -172,12 +168,13 @@ func (h *writeHandler) write(ctx context.Context, req *prompb.WriteRequest) (err }() b := labels.NewScratchBuilder(0) + var es []exemplar.Exemplar for _, ts := range req.Timeseries { ls := ts.ToLabels(&b, nil) // TODO(bwplotka): Even as per 1.0 spec, this should be a 400 error, while other samples are // potentially written. Perhaps unify with fixed writeV2 implementation a bit. - if !ls.Has(labels.MetricName) || !ls.IsValid(model.UTF8Validation) { + if !ls.Has(model.MetricNameLabel) || !ls.IsValid(model.UTF8Validation) { h.logger.Warn("Invalid metric names or labels", "got", ls.String()) samplesWithInvalidLabels++ continue @@ -187,26 +184,20 @@ func (h *writeHandler) write(ctx context.Context, req *prompb.WriteRequest) (err continue } - if err := h.appendV1Samples(app, ts.Samples, ls); err != nil { + es = es[:0] + for _, ep := range ts.Exemplars { + e := ep.ToExemplar(&b, nil) + es = append(es, e) + } + + outOfOrderExemplarErrs, err = h.appendV1Samples(app, ts.Samples, ls, es) + if err != nil { return err } samplesAppended += len(ts.Samples) - for _, ep := range ts.Exemplars { - e := ep.ToExemplar(&b, nil) - if _, err := app.AppendExemplar(0, ls, e); err != nil { - switch { - case errors.Is(err, storage.ErrOutOfOrderExemplar): - outOfOrderExemplarErrs++ - h.logger.Debug("Out of order exemplar", "series", ls.String(), "exemplar", fmt.Sprintf("%+v", e)) - default: - // Since exemplar storage is still experimental, we don't fail the request on ingestion errors - h.logger.Debug("Error while adding exemplar in AppendExemplar", "series", ls.String(), "exemplar", fmt.Sprintf("%+v", e), "err", err) - } - } - } - - if err = h.appendV1Histograms(app, ts.Histograms, ls); err != nil { + outOfOrderExemplarErrs, err = h.appendV1Histograms(app, ts.Histograms, ls, es) + if err != nil { return err } samplesAppended += len(ts.Histograms) @@ -221,43 +212,66 @@ func (h *writeHandler) write(ctx context.Context, req *prompb.WriteRequest) (err return nil } -func (h *writeHandler) appendV1Samples(app storage.Appender, ss []prompb.Sample, labels labels.Labels) error { +func (h *writeHandler) appendV1Samples(app storage.AppenderV2, ss []prompb.Sample, ls labels.Labels, es []exemplar.Exemplar) (outOfOrderExemplarErrs int, err error) { var ref storage.SeriesRef - var err error for _, s := range ss { - ref, err = app.Append(ref, labels, s.GetTimestamp(), s.GetValue()) + ref, err = app.Append(ref, ls, 0, s.GetTimestamp(), s.GetValue(), nil, nil, storage.AOptions{Exemplars: es}) if err != nil { if errors.Is(err, storage.ErrOutOfOrderSample) || errors.Is(err, storage.ErrOutOfBounds) || errors.Is(err, storage.ErrDuplicateSampleForTimestamp) { - h.logger.Error("Out of order sample from remote write", "err", err.Error(), "series", labels.String(), "timestamp", s.Timestamp) + h.logger.Error("Out of order sample from remote write", "err", err.Error(), "series", ls.String(), "timestamp", s.Timestamp) } - return err + var pErr *storage.AppendPartialError + if errors.As(err, &pErr) { + for _, e := range pErr.ExemplarErrors { + if errors.Is(e, storage.ErrOutOfOrderExemplar) { + outOfOrderExemplarErrs++ + h.logger.Debug("Out of order exemplar", "series", ls.String()) + continue + } + // Since exemplar storage is still experimental, we don't fail the request on ingestion errors + h.logger.Debug("Error while adding exemplar in AppendExemplar", "series", ls.String(), "err", err) + } + // Still claim success and continue. + continue + } + return outOfOrderExemplarErrs, err } } - return nil + return outOfOrderExemplarErrs, nil } -func (h *writeHandler) appendV1Histograms(app storage.Appender, hh []prompb.Histogram, labels labels.Labels) error { - var err error +func (h *writeHandler) appendV1Histograms(app storage.AppenderV2, hh []prompb.Histogram, ls labels.Labels, es []exemplar.Exemplar) (outOfOrderExemplarErrs int, err error) { + var ref storage.SeriesRef for _, hp := range hh { - if hp.IsFloatHistogram() { - _, err = app.AppendHistogram(0, labels, hp.Timestamp, nil, hp.ToFloatHistogram()) - } else { - _, err = app.AppendHistogram(0, labels, hp.Timestamp, hp.ToIntHistogram(), nil) - } + ref, err = app.Append(ref, ls, 0, hp.GetTimestamp(), 0, hp.ToIntHistogram(), hp.ToFloatHistogram(), storage.AOptions{Exemplars: es}) if err != nil { - // Although AppendHistogram does not currently return ErrDuplicateSampleForTimestamp there is + // Although Append does not currently return ErrDuplicateSampleForTimestamp there is // a note indicating its inclusion in the future. if errors.Is(err, storage.ErrOutOfOrderSample) || errors.Is(err, storage.ErrOutOfBounds) || errors.Is(err, storage.ErrDuplicateSampleForTimestamp) { - h.logger.Error("Out of order histogram from remote write", "err", err.Error(), "series", labels.String(), "timestamp", hp.Timestamp) + h.logger.Error("Out of order histogram from remote write", "err", err.Error(), "series", ls.String(), "timestamp", hp.Timestamp) } - return err + var pErr *storage.AppendPartialError + if errors.As(err, &pErr) { + for _, e := range pErr.ExemplarErrors { + if errors.Is(e, storage.ErrOutOfOrderExemplar) { + outOfOrderExemplarErrs++ + h.logger.Debug("Out of order exemplar", "series", ls.String()) + continue + } + // Since exemplar storage is still experimental, we don't fail the request on ingestion errors + h.logger.Debug("Error while adding exemplar in AppendExemplar", "series", ls.String(), "err", err) + } + // Still claim success and continue. + continue + } + return outOfOrderExemplarErrs, err } } - return nil + return outOfOrderExemplarErrs, nil } // writeV2 is similar to write, but it works with v2 proto message, @@ -270,9 +284,9 @@ func (h *writeHandler) appendV1Histograms(app storage.Appender, hh []prompb.Hist // NOTE(bwplotka): TSDB storage is NOT idempotent, so we don't allow "partial retry-able" errors. // Once we have 5xx type of error, we immediately stop and rollback all appends. func (h *writeHandler) writeV2(ctx context.Context, req *writev2.Request) (_ remoteapi.WriteResponseStats, errHTTPCode int, _ error) { - app := &remoteWriteAppender{ - Appender: h.appendable.Appender(ctx), - maxTime: timestamp.FromTime(time.Now().Add(maxAheadTime)), + app := &validationAppender{ + AppenderV2: h.appendable.AppenderV2(ctx), + maxTime: timestamp.FromTime(time.Now().Add(maxAheadTime)), } s := remoteapi.WriteResponseStats{} @@ -306,10 +320,11 @@ func (h *writeHandler) writeV2(ctx context.Context, req *writev2.Request) (_ rem return s, 0, nil } -func (h *writeHandler) appendV2(app storage.Appender, req *writev2.Request, rs *remoteapi.WriteResponseStats) (samplesWithoutMetadata, errHTTPCode int, err error) { +func (h *writeHandler) appendV2(app storage.AppenderV2, req *writev2.Request, rs *remoteapi.WriteResponseStats) (samplesWithoutMetadata, errHTTPCode int, err error) { var ( badRequestErrs []error outOfOrderExemplarErrs, samplesWithInvalidLabels int + es []exemplar.Exemplar b = labels.NewScratchBuilder(0) ) @@ -322,6 +337,11 @@ func (h *writeHandler) appendV2(app storage.Appender, req *writev2.Request, rs * } m := ts.ToMetadata(req.Symbols) + if m.IsEmpty() { + // Account for missing metadata this TimeSeries. + samplesWithoutMetadata += rs.AllSamples() + } + if h.enableTypeAndUnitLabels && (m.Type != model.MetricTypeUnknown || m.Unit != "") { slb := labels.NewScratchBuilder(ls.Len() + 2) // +2 for __type__ and __unit__ ls.Range(func(l labels.Label) { @@ -339,7 +359,7 @@ func (h *writeHandler) appendV2(app storage.Appender, req *writev2.Request, rs * // Validate series labels early. // NOTE(bwplotka): While spec allows UTF-8, Prometheus Receiver may impose // specific limits and follow https://prometheus.io/docs/specs/remote_write_spec_2_0/#invalid-samples case. - if !ls.Has(labels.MetricName) || !ls.IsValid(model.UTF8Validation) { + if !ls.Has(model.MetricNameLabel) || !ls.IsValid(model.UTF8Validation) { badRequestErrs = append(badRequestErrs, fmt.Errorf("invalid metric name or labels, got %v", ls.String())) samplesWithInvalidLabels += len(ts.Samples) + len(ts.Histograms) continue @@ -354,22 +374,32 @@ func (h *writeHandler) appendV2(app storage.Appender, req *writev2.Request, rs * badRequestErrs = append(badRequestErrs, fmt.Errorf("TimeSeries must contain at least one sample or histogram for series %v", ls.String())) continue } + // Validate that TimeSeries does not have both; it's against the spec e.g. where to attach exemplars to? + if len(ts.Samples) > 0 && len(ts.Histograms) > 0 { + badRequestErrs = append(badRequestErrs, fmt.Errorf("TimeSeries must contain either samples or histograms for series %v not both", ls.String())) + continue + } - allSamplesSoFar := rs.AllSamples() var ref storage.SeriesRef - for _, s := range ts.Samples { - if h.ingestSTZeroSample && s.StartTimestamp != 0 && s.Timestamp != 0 { - ref, err = app.AppendSTZeroSample(ref, ls, s.Timestamp, s.StartTimestamp) - // We treat OOO errors specially as it's a common scenario given: - // * We can't tell if ST was already ingested in a previous request. - // * We don't check if ST changed for stream of samples (we typically have one though), - // as it's checked in the AppendSTZeroSample reliably. - if err != nil && !errors.Is(err, storage.ErrOutOfOrderST) { - h.logger.Debug("Error when appending ST from remote write request", "err", err, "series", ls.String(), "start_timestamp", s.StartTimestamp, "timestamp", s.Timestamp) - } - } - ref, err = app.Append(ref, ls, s.GetTimestamp(), s.GetValue()) + // Attach potential exemplars to append. + es = es[:0] + for _, ep := range ts.Exemplars { + e, err := ep.ToExemplar(&b, req.Symbols) + if err != nil { + badRequestErrs = append(badRequestErrs, fmt.Errorf("parsing exemplar for series %v: %w", ls.String(), err)) + continue + } + es = append(es, e) + } + appOpts := storage.AppendV2Options{ + Metadata: m, + Exemplars: es, + } + rs.Exemplars += len(appOpts.Exemplars) // Rejection is accounted later on. + + for _, s := range ts.Samples { + ref, err = app.Append(ref, ls, s.GetStartTimestamp(), s.GetTimestamp(), s.GetValue(), nil, nil, appOpts) if err == nil { rs.Samples++ continue @@ -384,26 +414,29 @@ func (h *writeHandler) appendV2(app storage.Appender, req *writev2.Request, rs * badRequestErrs = append(badRequestErrs, fmt.Errorf("%w for series %v", err, ls.String())) continue } + var pErr *storage.AppendPartialError + if errors.As(err, &pErr) { + for _, e := range pErr.ExemplarErrors { + rs.Exemplars-- + if errors.Is(e, storage.ErrOutOfOrderExemplar) { + outOfOrderExemplarErrs++ // Maintain old metrics, but technically not needed, given we fail here. + h.logger.Error("Out of order exemplar", "err", err.Error(), "series", ls.String()) + badRequestErrs = append(badRequestErrs, fmt.Errorf("%w for series %v", err, ls.String())) + continue + } + // Since exemplar storage is still experimental, we don't fail or check other errors. + // Debug log is emitted in TSDB already. + } + // Still claim success and continue. + rs.Samples++ + continue + } return 0, http.StatusInternalServerError, err } // Native Histograms. for _, hp := range ts.Histograms { - if h.ingestSTZeroSample && hp.StartTimestamp != 0 && hp.Timestamp != 0 { - ref, err = h.handleHistogramZeroSample(app, ref, ls, hp, hp.StartTimestamp) - // We treat OOO errors specially as it's a common scenario given: - // * We can't tell if ST was already ingested in a previous request. - // * We don't check if ST changed for stream of samples (we typically have one though), - // as it's checked in the ingestSTZeroSample reliably. - if err != nil && !errors.Is(err, storage.ErrOutOfOrderST) { - h.logger.Debug("Error when appending ST from remote write request", "err", err, "series", ls.String(), "start_timestamp", hp.StartTimestamp, "timestamp", hp.Timestamp) - } - } - if hp.IsFloatHistogram() { - ref, err = app.AppendHistogram(ref, ls, hp.Timestamp, nil, hp.ToFloatHistogram()) - } else { - ref, err = app.AppendHistogram(ref, ls, hp.Timestamp, hp.ToIntHistogram(), nil) - } + ref, err = app.Append(ref, ls, hp.GetStartTimestamp(), hp.GetTimestamp(), 0, hp.ToIntHistogram(), hp.ToFloatHistogram(), appOpts) if err == nil { rs.Histograms++ continue @@ -424,43 +457,25 @@ func (h *writeHandler) appendV2(app storage.Appender, req *writev2.Request, rs * badRequestErrs = append(badRequestErrs, fmt.Errorf("%w for series %v", err, ls.String())) continue } + var pErr *storage.AppendPartialError + if errors.As(err, &pErr) { + for _, e := range pErr.ExemplarErrors { + rs.Exemplars-- + if errors.Is(e, storage.ErrOutOfOrderExemplar) { + outOfOrderExemplarErrs++ // Maintain old metrics, but technically not needed, given we fail here. + h.logger.Error("Out of order exemplar", "err", err.Error(), "series", ls.String()) + badRequestErrs = append(badRequestErrs, fmt.Errorf("%w for series %v", err, ls.String())) + continue + } + // Since exemplar storage is still experimental, we don't fail or check other errors. + // Debug log is emitted in TSDB already. + } + // Still claim success and continue. + rs.Histograms++ + continue + } return 0, http.StatusInternalServerError, err } - - // Exemplars. - for _, ep := range ts.Exemplars { - e, err := ep.ToExemplar(&b, req.Symbols) - if err != nil { - badRequestErrs = append(badRequestErrs, fmt.Errorf("parsing exemplar for series %v: %w", ls.String(), err)) - continue - } - ref, err = app.AppendExemplar(ref, ls, e) - if err == nil { - rs.Exemplars++ - continue - } - // Handle append error. - if errors.Is(err, storage.ErrOutOfOrderExemplar) { - outOfOrderExemplarErrs++ // Maintain old metrics, but technically not needed, given we fail here. - h.logger.Error("Out of order exemplar", "err", err.Error(), "series", ls.String(), "exemplar", fmt.Sprintf("%+v", e)) - badRequestErrs = append(badRequestErrs, fmt.Errorf("%w for series %v", err, ls.String())) - continue - } - // TODO(bwplotka): Add strict mode which would trigger rollback of everything if needed. - // For now we keep the previously released flow (just error not debug leve) of dropping them without rollback and 5xx. - h.logger.Error("failed to ingest exemplar, emitting error log, but no error for PRW caller", "err", err.Error(), "series", ls.String(), "exemplar", fmt.Sprintf("%+v", e)) - } - - // Only update metadata in WAL if the metadata-wal-records feature is enabled. - // Without this feature, metadata is not persisted to WAL. - if h.appendMetadata { - if _, err = app.UpdateMetadata(ref, ls, m); err != nil { - h.logger.Debug("error while updating metadata from remote write", "err", err) - // Metadata is attached to each series, so since Prometheus does not reject sample without metadata information, - // we don't report remote write error either. We increment metric instead. - samplesWithoutMetadata += rs.AllSamples() - allSamplesSoFar - } - } } if outOfOrderExemplarErrs > 0 { @@ -499,16 +514,11 @@ type OTLPOptions struct { LookbackDelta time.Duration // Add type and unit labels to the metrics. EnableTypeAndUnitLabels bool - // IngestSTZeroSample enables writing zero samples based on the start time - // of metrics. - IngestSTZeroSample bool - // AppendMetadata enables writing metadata to WAL when metadata-wal-records feature is enabled. - AppendMetadata bool } // NewOTLPWriteHandler creates a http.Handler that accepts OTLP write requests and // writes them to the provided appendable. -func NewOTLPWriteHandler(logger *slog.Logger, reg prometheus.Registerer, appendable storage.Appendable, configFunc func() config.Config, opts OTLPOptions) http.Handler { +func NewOTLPWriteHandler(logger *slog.Logger, reg prometheus.Registerer, appendable storage.AppendableV2, configFunc func() config.Config, opts OTLPOptions) http.Handler { if opts.NativeDelta && opts.ConvertDelta { // This should be validated when iterating through feature flags, so not expected to fail here. panic("cannot enable native delta ingestion and delta2cumulative conversion at the same time") @@ -520,11 +530,7 @@ func NewOTLPWriteHandler(logger *slog.Logger, reg prometheus.Registerer, appenda config: configFunc, allowDeltaTemporality: opts.NativeDelta, lookbackDelta: opts.LookbackDelta, - ingestSTZeroSample: opts.IngestSTZeroSample, enableTypeAndUnitLabels: opts.EnableTypeAndUnitLabels, - appendMetadata: opts.AppendMetadata, - // Register metrics. - metrics: otlptranslator.NewCombinedAppenderMetrics(reg), } wh := &otlpWriteHandler{logger: logger, defaultConsumer: ex} @@ -559,26 +565,45 @@ func NewOTLPWriteHandler(logger *slog.Logger, reg prometheus.Registerer, appenda type rwExporter struct { logger *slog.Logger - appendable storage.Appendable + appendable storage.AppendableV2 config func() config.Config allowDeltaTemporality bool lookbackDelta time.Duration - ingestSTZeroSample bool enableTypeAndUnitLabels bool - appendMetadata bool - - // Metrics. - metrics otlptranslator.CombinedAppenderMetrics } func (rw *rwExporter) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) error { otlpCfg := rw.config().OTLPConfig - app := &remoteWriteAppender{ - Appender: rw.appendable.Appender(ctx), - maxTime: timestamp.FromTime(time.Now().Add(maxAheadTime)), + app := &validationAppender{ + AppenderV2: rw.appendable.AppenderV2(ctx), + maxTime: timestamp.FromTime(time.Now().Add(maxAheadTime)), } - combinedAppender := otlptranslator.NewCombinedAppender(app, rw.logger, rw.ingestSTZeroSample, rw.appendMetadata, rw.metrics) - converter := otlptranslator.NewPrometheusConverter(combinedAppender) + + // NOTE(bwplotka): When switching to AppenderV2 I skipped 2 things: + // * Metrics + // // TODO: Add, likely in a single place in metrics_to_prw.go + // samplesAppendedWithoutMetadata: promauto.With(reg).NewCounter(prometheus.CounterOpts{ + // Namespace: "prometheus", + // Subsystem: "api", + // Name: "otlp_appended_samples_without_metadata_total", + // Help: "The total number of samples ingested from OTLP without corresponding metadata.", + // }), + // // TODO: Add using storage.AppenderPartialError + // outOfOrderExemplars: promauto.With(reg).NewCounter(prometheus.CounterOpts{ + // Namespace: "prometheus", + // Subsystem: "api", + // Name: "otlp_out_of_order_exemplars_total", + // Help: "The total number of received OTLP exemplars which were rejected because they were out of order.", + // }), + // } + // * this odd ref cache. This one I propose to skip until we know we need it for efficiency reasons. + // As a part of a single OTLP message, do we even envision ANY ref to be shared? (it's only one sample per series, no? + // + // // Used to ensure we only update metadata and created timestamps once, and to share storage.SeriesRefs. + // // To detect hash collision it also stores the labels. + // // There is no overflow/conflict list, the TSDB will handle that part. + // refs map[uint64]seriesRef + converter := otlptranslator.NewPrometheusConverter(app) annots, err := converter.FromMetrics(ctx, md, otlptranslator.Settings{ AddMetricSuffixes: otlpCfg.TranslationStrategy.ShouldAddSuffixes(), AllowUTF8: !otlpCfg.TranslationStrategy.ShouldEscape(), @@ -678,55 +703,26 @@ func hasDelta(md pmetric.Metrics) bool { return false } -type remoteWriteAppender struct { - storage.Appender +type validationAppender struct { + storage.AppenderV2 maxTime int64 } -func (app *remoteWriteAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64, v float64) (storage.SeriesRef, error) { - if t > app.maxTime { - return 0, fmt.Errorf("%w: timestamp is too far in the future", storage.ErrOutOfBounds) - } - - ref, err := app.Appender.Append(ref, lset, t, v) - if err != nil { - return 0, err - } - return ref, nil -} - -func (app *remoteWriteAppender) AppendHistogram(ref storage.SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { - var err error +func (app *validationAppender) Append(ref storage.SeriesRef, ls labels.Labels, st, t int64, v float64, h *histogram.Histogram, fh *histogram.FloatHistogram, opts storage.AOptions) (storage.SeriesRef, error) { if t > app.maxTime { return 0, fmt.Errorf("%w: timestamp is too far in the future", storage.ErrOutOfBounds) } if h != nil && histogram.IsExponentialSchemaReserved(h.Schema) && h.Schema > histogram.ExponentialSchemaMax { - if err = h.ReduceResolution(histogram.ExponentialSchemaMax); err != nil { + if err := h.ReduceResolution(histogram.ExponentialSchemaMax); err != nil { return 0, err } } if fh != nil && histogram.IsExponentialSchemaReserved(fh.Schema) && fh.Schema > histogram.ExponentialSchemaMax { - if err = fh.ReduceResolution(histogram.ExponentialSchemaMax); err != nil { + if err := fh.ReduceResolution(histogram.ExponentialSchemaMax); err != nil { return 0, err } } - - if ref, err = app.Appender.AppendHistogram(ref, l, t, h, fh); err != nil { - return 0, err - } - return ref, nil -} - -func (app *remoteWriteAppender) AppendExemplar(ref storage.SeriesRef, l labels.Labels, e exemplar.Exemplar) (storage.SeriesRef, error) { - if e.Ts > app.maxTime { - return 0, fmt.Errorf("%w: timestamp is too far in the future", storage.ErrOutOfBounds) - } - - ref, err := app.Appender.AppendExemplar(ref, l, e) - if err != nil { - return 0, err - } - return ref, nil + return app.AppenderV2.Append(ref, ls, st, t, v, h, fh, opts) } diff --git a/storage/remote/write_handler_test.go b/storage/remote/write_handler_test.go index afc0d985ff..e888cb0de2 100644 --- a/storage/remote/write_handler_test.go +++ b/storage/remote/write_handler_test.go @@ -31,9 +31,9 @@ import ( "github.com/google/go-cmp/cmp" remoteapi "github.com/prometheus/client_golang/exp/api/remote" "github.com/prometheus/common/promslog" + "github.com/prometheus/prometheus/util/teststorage" "github.com/stretchr/testify/require" - "github.com/prometheus/prometheus/model/exemplar" "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/metadata" @@ -129,8 +129,7 @@ func TestRemoteWriteHandlerHeadersHandling_V1Message(t *testing.T) { req.Header.Set(k, v) } - appendable := &mockAppendable{} - handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false) + handler := NewWriteHandler(promslog.NewNopLogger(), nil, &mockAppendable{}, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false) recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, req) @@ -236,8 +235,11 @@ func TestRemoteWriteHandlerHeadersHandling_V2Message(t *testing.T) { req.Header.Set(k, v) } - appendable := &mockAppendable{} - handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, false, false, false) + s := teststorage.New(t) + t.Cleanup(func() { _ = s.Close() }) + + appendable := s //&mockAppendable{} + handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, false) recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, req) @@ -253,9 +255,9 @@ func TestRemoteWriteHandlerHeadersHandling_V2Message(t *testing.T) { // Invalid request case - no samples should be written. require.Equal(t, tc.expectedError, strings.TrimSpace(string(out))) - require.Empty(t, appendable.samples) - require.Empty(t, appendable.histograms) - require.Empty(t, appendable.exemplars) + // require.Empty(t, appendable.samples) + // require.Empty(t, appendable.histograms) + // require.Empty(t, appendable.exemplars) }) } @@ -272,7 +274,7 @@ func TestRemoteWriteHandlerHeadersHandling_V2Message(t *testing.T) { } appendable := &mockAppendable{} - handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, false, false, false) + handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, false) recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, req) @@ -301,7 +303,7 @@ func TestRemoteWriteHandler_V1Message(t *testing.T) { // in Prometheus, so keeping like this to not break existing 1.0 clients. appendable := &mockAppendable{} - handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false) + handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false) recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, req) @@ -314,25 +316,18 @@ func TestRemoteWriteHandler_V1Message(t *testing.T) { j := 0 k := 0 for _, ts := range writeRequestFixture.Timeseries { - labels := ts.ToLabels(&b, nil) + ls := ts.ToLabels(&b, nil) for _, s := range ts.Samples { - requireEqual(t, mockSample{labels, s.Timestamp, s.Value}, appendable.samples[i]) + requireEqual(t, mockSample{ls, metadata.Metadata{}, 0, s.Timestamp, s.Value}, appendable.samples[i]) i++ } for _, e := range ts.Exemplars { exemplarLabels := e.ToExemplar(&b, nil).Labels - requireEqual(t, mockExemplar{labels, exemplarLabels, e.Timestamp, e.Value}, appendable.exemplars[j]) + requireEqual(t, mockExemplar{ls, exemplarLabels, e.Timestamp, e.Value}, appendable.exemplars[j]) j++ } for _, hp := range ts.Histograms { - if hp.IsFloatHistogram() { - fh := hp.ToFloatHistogram() - requireEqual(t, mockHistogram{labels, hp.Timestamp, nil, fh}, appendable.histograms[k]) - } else { - h := hp.ToIntHistogram() - requireEqual(t, mockHistogram{labels, hp.Timestamp, h, nil}, appendable.histograms[k]) - } - + requireEqual(t, mockHistogram{ls, metadata.Metadata{}, 0, hp.Timestamp, hp.ToIntHistogram(), hp.ToFloatHistogram()}, appendable.histograms[k]) k++ } } @@ -356,26 +351,15 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) { expectedCode int expectedRespBody string - commitErr error - appendSampleErr error - appendSTZeroSampleErr error - appendHistogramErr error - appendExemplarErr error - updateMetadataErr error + commitErr error + appendSampleErr error + appendExemplarErr error - ingestSTZeroSample bool enableTypeAndUnitLabels bool - appendMetadata bool expectedLabels labels.Labels // For verifying type/unit labels }{ { - desc: "All timeseries accepted/ct_enabled", - input: writeV2RequestFixture.Timeseries, - expectedCode: http.StatusNoContent, - ingestSTZeroSample: true, - }, - { - desc: "All timeseries accepted/ct_disabled", + desc: "All timeseries accepted", input: writeV2RequestFixture.Timeseries, expectedCode: http.StatusNoContent, }, @@ -469,15 +453,25 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) { expectedRespBody: "out of order sample for series {__name__=\"test_metric1\", b=\"c\", baz=\"qux\", d=\"e\", foo=\"bar\"}\n", }, { - desc: "Partial write; first series with one dup histogram sample", + desc: "Partial write; 3rd series with one dup histogram sample", input: func() []writev2.TimeSeries { f := proto.Clone(writeV2RequestFixture).(*writev2.Request) - f.Timeseries[0].Histograms = append(f.Timeseries[0].Histograms, f.Timeseries[0].Histograms[len(f.Timeseries[0].Histograms)-1]) + f.Timeseries[2].Histograms = append(f.Timeseries[2].Histograms, f.Timeseries[2].Histograms[len(f.Timeseries[2].Histograms)-1]) return f.Timeseries }(), expectedCode: http.StatusBadRequest, expectedRespBody: "duplicate sample for timestamp for series {__name__=\"test_metric1\", b=\"c\", baz=\"qux\", d=\"e\", foo=\"bar\"}\n", }, + { + desc: "Partial write; first series have both sample and histogram", + input: func() []writev2.TimeSeries { + f := proto.Clone(writeV2RequestFixture).(*writev2.Request) + f.Timeseries[0].Histograms = append(f.Timeseries[0].Histograms, writev2.FromFloatHistogram(1, testHistogram.ToFloat(nil))) + return f.Timeseries + }(), + expectedCode: http.StatusBadRequest, + expectedRespBody: "TBDn", + }, // Non retriable errors from various parts. { desc: "Internal sample append error; rollback triggered", @@ -487,14 +481,6 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) { expectedCode: http.StatusInternalServerError, expectedRespBody: "some sample internal append error\n", }, - { - desc: "Internal histogram sample append error; rollback triggered", - input: writeV2RequestFixture.Timeseries, - appendHistogramErr: errors.New("some histogram sample internal append error"), - - expectedCode: http.StatusInternalServerError, - expectedRespBody: "some histogram sample internal append error\n", - }, { desc: "Partial write; skipped exemplar; exemplar storage errs are noop", input: writeV2RequestFixture.Timeseries, @@ -502,13 +488,6 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) { expectedCode: http.StatusNoContent, }, - { - desc: "Partial write; skipped metadata; metadata storage errs are noop", - input: writeV2RequestFixture.Timeseries, - updateMetadataErr: errors.New("some metadata update error"), - - expectedCode: http.StatusNoContent, - }, { desc: "Internal commit error; rollback triggered", input: writeV2RequestFixture.Timeseries, @@ -627,7 +606,6 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) { }(), expectedCode: http.StatusNoContent, enableTypeAndUnitLabels: false, - appendMetadata: false, expectedLabels: labels.FromStrings("__name__", "test_metric_wal", "instance", "localhost"), }, { @@ -699,23 +677,20 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) { req.Header.Set(RemoteWriteVersionHeader, RemoteWriteVersion20HeaderValue) appendable := &mockAppendable{ - commitErr: tc.commitErr, - appendSampleErr: tc.appendSampleErr, - appendSTZeroSampleErr: tc.appendSTZeroSampleErr, - appendHistogramErr: tc.appendHistogramErr, - appendExemplarErr: tc.appendExemplarErr, - updateMetadataErr: tc.updateMetadataErr, + commitErr: tc.commitErr, + appendSampleErr: tc.appendSampleErr, + appendExemplarErr: tc.appendExemplarErr, } - handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, tc.ingestSTZeroSample, tc.enableTypeAndUnitLabels, tc.appendMetadata) + handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, tc.enableTypeAndUnitLabels) recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, req) resp := recorder.Result() - require.Equal(t, tc.expectedCode, resp.StatusCode) respBody, err := io.ReadAll(resp.Body) require.NoError(t, err) require.Equal(t, tc.expectedRespBody, string(respBody)) + require.Equal(t, tc.expectedCode, resp.StatusCode) if tc.expectedCode == http.StatusInternalServerError { // We don't expect writes for partial writes with retry-able code. @@ -726,7 +701,6 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) { require.Empty(t, appendable.samples) require.Empty(t, appendable.histograms) require.Empty(t, appendable.exemplars) - require.Empty(t, appendable.metadata) return } @@ -748,37 +722,21 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) { // Double check what was actually appended. var ( - b = labels.NewScratchBuilder(0) - i, j, k, m int + b = labels.NewScratchBuilder(0) + i, j, k int ) for _, ts := range writeV2RequestFixture.Timeseries { + expectedMeta := ts.ToMetadata(writeV2RequestFixture.Symbols) + ls, err := ts.ToLabels(&b, writeV2RequestFixture.Symbols) require.NoError(t, err) for _, s := range ts.Samples { - if s.StartTimestamp != 0 && tc.ingestSTZeroSample { - requireEqual(t, mockSample{ls, s.StartTimestamp, 0}, appendable.samples[i]) - i++ - } - requireEqual(t, mockSample{ls, s.Timestamp, s.Value}, appendable.samples[i]) + requireEqual(t, mockSample{ls, expectedMeta, s.StartTimestamp, s.Timestamp, s.Value}, appendable.samples[i]) i++ } for _, hp := range ts.Histograms { - if hp.IsFloatHistogram() { - fh := hp.ToFloatHistogram() - if hp.StartTimestamp != 0 && tc.ingestSTZeroSample { - requireEqual(t, mockHistogram{ls, hp.StartTimestamp, nil, &histogram.FloatHistogram{}}, appendable.histograms[k]) - k++ - } - requireEqual(t, mockHistogram{ls, hp.Timestamp, nil, fh}, appendable.histograms[k]) - } else { - h := hp.ToIntHistogram() - if hp.StartTimestamp != 0 && tc.ingestSTZeroSample { - requireEqual(t, mockHistogram{ls, hp.StartTimestamp, &histogram.Histogram{}, nil}, appendable.histograms[k]) - k++ - } - requireEqual(t, mockHistogram{ls, hp.Timestamp, h, nil}, appendable.histograms[k]) - } + requireEqual(t, mockHistogram{ls, expectedMeta, hp.StartTimestamp, hp.Timestamp, hp.ToIntHistogram(), hp.ToFloatHistogram()}, appendable.histograms[k]) k++ } if tc.appendExemplarErr == nil { @@ -790,16 +748,6 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) { j++ } } - if tc.appendMetadata && tc.updateMetadataErr == nil { - expectedMeta := ts.ToMetadata(writeV2RequestFixture.Symbols) - requireEqual(t, mockMetadata{ls, expectedMeta}, appendable.metadata[m]) - m++ - } - } - - // Verify that when the feature flag is disabled, no metadata is stored in WAL. - if !tc.appendMetadata { - require.Empty(t, appendable.metadata, "metadata should not be stored when appendMetadata (metadata-wal-records) is false") } }) } @@ -880,7 +828,7 @@ func TestRemoteWriteHandler_V2Message_NoDuplicateTypeAndUnitLabels(t *testing.T) req.Header.Set(RemoteWriteVersionHeader, RemoteWriteVersion20HeaderValue) appendable := &mockAppendable{} - handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, false, true, false) + handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, true) recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, req) @@ -928,8 +876,8 @@ func TestOutOfOrderSample_V1Message(t *testing.T) { req, err := http.NewRequest(http.MethodPost, "", bytes.NewReader(payload)) require.NoError(t, err) - appendable := &mockAppendable{latestSample: map[uint64]int64{labels.FromStrings("__name__", "test_metric").Hash(): 100}} - handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false) + appendable := &mockAppendable{latestTs: map[uint64]int64{labels.FromStrings("__name__", "test_metric").Hash(): 100}} + handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false) recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, req) @@ -970,8 +918,8 @@ func TestOutOfOrderExemplar_V1Message(t *testing.T) { req, err := http.NewRequest(http.MethodPost, "", bytes.NewReader(payload)) require.NoError(t, err) - appendable := &mockAppendable{latestSample: map[uint64]int64{labels.FromStrings("__name__", "test_metric").Hash(): 100}} - handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false) + appendable := &mockAppendable{latestTs: map[uint64]int64{labels.FromStrings("__name__", "test_metric").Hash(): 100}} + handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false) recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, req) @@ -1008,8 +956,8 @@ func TestOutOfOrderHistogram_V1Message(t *testing.T) { req, err := http.NewRequest(http.MethodPost, "", bytes.NewReader(payload)) require.NoError(t, err) - appendable := &mockAppendable{latestSample: map[uint64]int64{labels.FromStrings("__name__", "test_metric").Hash(): 100}} - handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false) + appendable := &mockAppendable{latestTs: map[uint64]int64{labels.FromStrings("__name__", "test_metric").Hash(): 100}} + handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false) recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, req) @@ -1059,7 +1007,7 @@ func BenchmarkRemoteWriteHandler(b *testing.B) { for _, tc := range testCases { b.Run(tc.name, func(b *testing.B) { appendable := &mockAppendable{} - handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{tc.protoFormat}, false, false, false) + handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{tc.protoFormat}, false) b.ResetTimer() for b.Loop() { b.StopTimer() @@ -1084,7 +1032,7 @@ func TestCommitErr_V1Message(t *testing.T) { require.NoError(t, err) appendable := &mockAppendable{commitErr: errors.New("commit error")} - handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false) + handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false) recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, req) @@ -1150,7 +1098,7 @@ func TestHistogramValidationErrorHandling(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { require.NoError(t, db.Close()) }) - handler := NewWriteHandler(promslog.NewNopLogger(), nil, db.Head(), []remoteapi.WriteMessageType{protoMsg}, false, false, false) + handler := NewWriteHandler(promslog.NewNopLogger(), nil, db.Head(), []remoteapi.WriteMessageType{protoMsg}, false) recorder := httptest.NewRecorder() var buf []byte @@ -1195,7 +1143,7 @@ func TestCommitErr_V2Message(t *testing.T) { req.Header.Set(RemoteWriteVersionHeader, RemoteWriteVersion20HeaderValue) appendable := &mockAppendable{commitErr: errors.New("commit error")} - handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, false, false, false) + handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{remoteapi.WriteV2MessageType}, false) recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, req) @@ -1222,7 +1170,7 @@ func BenchmarkRemoteWriteOOOSamples(b *testing.B) { require.NoError(b, db.Close()) }) // TODO: test with other proto format(s) - handler := NewWriteHandler(promslog.NewNopLogger(), nil, db.Head(), []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false, false, false) + handler := NewWriteHandler(promslog.NewNopLogger(), nil, db.Head(), []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType}, false) buf, _, _, err := buildWriteRequest(nil, genSeriesWithSample(1000, 200*time.Minute.Milliseconds()), nil, nil, nil, nil, "snappy") require.NoError(b, err) @@ -1267,29 +1215,26 @@ func genSeriesWithSample(numSeries int, ts int64) []prompb.TimeSeries { return series } +// TODO(bwplotka): We have 3 mocks of appender at this point. Consolidate? type mockAppendable struct { - latestSample map[uint64]int64 - samples []mockSample - latestExemplar map[uint64]int64 - exemplars []mockExemplar - latestHistogram map[uint64]int64 - latestFloatHist map[uint64]int64 - histograms []mockHistogram - metadata []mockMetadata + latestTs map[uint64]int64 + samples []mockSample + exemplars []mockExemplar + latestExemplarTs map[uint64]int64 + histograms []mockHistogram // optional errors to inject. - commitErr error - appendSampleErr error - appendSTZeroSampleErr error - appendHistogramErr error - appendExemplarErr error - updateMetadataErr error + commitErr error + appendSampleErr error + appendHistogramErr error + appendExemplarErr error } type mockSample struct { - l labels.Labels - t int64 - v float64 + l labels.Labels + m metadata.Metadata + st, t int64 + v float64 } type mockExemplar struct { @@ -1300,15 +1245,11 @@ type mockExemplar struct { } type mockHistogram struct { - l labels.Labels - t int64 - h *histogram.Histogram - fh *histogram.FloatHistogram -} - -type mockMetadata struct { - l labels.Labels - m metadata.Metadata + l labels.Labels + m metadata.Metadata + st, t int64 + h *histogram.Histogram + fh *histogram.FloatHistogram } // Wrapper to instruct go-cmp package to compare a list of structs with unexported fields. @@ -1316,36 +1257,26 @@ func requireEqual(t *testing.T, expected, actual any, msgAndArgs ...any) { t.Helper() testutil.RequireEqualWithOptions(t, expected, actual, - []cmp.Option{cmp.AllowUnexported(mockSample{}), cmp.AllowUnexported(mockExemplar{}), cmp.AllowUnexported(mockHistogram{}), cmp.AllowUnexported(mockMetadata{})}, + []cmp.Option{cmp.AllowUnexported(), cmp.AllowUnexported(mockSample{}, mockExemplar{}, mockHistogram{})}, msgAndArgs...) } -func (m *mockAppendable) Appender(context.Context) storage.Appender { - if m.latestSample == nil { - m.latestSample = map[uint64]int64{} +func (m *mockAppendable) AppenderV2(context.Context) storage.AppenderV2 { + if m.latestTs == nil { + m.latestTs = map[uint64]int64{} } - if m.latestHistogram == nil { - m.latestHistogram = map[uint64]int64{} - } - if m.latestFloatHist == nil { - m.latestFloatHist = map[uint64]int64{} - } - if m.latestExemplar == nil { - m.latestExemplar = map[uint64]int64{} + if m.latestExemplarTs == nil { + m.latestExemplarTs = map[uint64]int64{} } return m } -func (*mockAppendable) SetOptions(*storage.AppendOptions) { - panic("unimplemented") -} - -func (m *mockAppendable) Append(_ storage.SeriesRef, l labels.Labels, t int64, v float64) (storage.SeriesRef, error) { +func (m *mockAppendable) Append(_ storage.SeriesRef, ls labels.Labels, st, t int64, v float64, h *histogram.Histogram, fh *histogram.FloatHistogram, opts storage.AOptions) (storage.SeriesRef, error) { if m.appendSampleErr != nil { return 0, m.appendSampleErr } - hash := l.Hash() - latestTs := m.latestSample[hash] + ref := ls.Hash() + latestTs := m.latestTs[ref] if t < latestTs { return 0, storage.ErrOutOfOrderSample } @@ -1353,16 +1284,45 @@ func (m *mockAppendable) Append(_ storage.SeriesRef, l labels.Labels, t int64, v return 0, storage.ErrDuplicateSampleForTimestamp } - if l.IsEmpty() { + if ls.IsEmpty() { return 0, tsdb.ErrInvalidSample } - if _, hasDuplicates := l.HasDuplicateLabelNames(); hasDuplicates { + if _, hasDuplicates := ls.HasDuplicateLabelNames(); hasDuplicates { return 0, tsdb.ErrInvalidSample } - m.latestSample[hash] = t - m.samples = append(m.samples, mockSample{l, t, v}) - return storage.SeriesRef(hash), nil + m.latestTs[ref] = t + switch { + case h != nil, fh != nil: + m.histograms = append(m.histograms, mockHistogram{ls, opts.Metadata, st, t, h, fh}) + default: + m.samples = append(m.samples, mockSample{ls, opts.Metadata, st, t, v}) + } + + var exErrs []error + if m.appendExemplarErr != nil { + exErrs = append(exErrs, m.appendExemplarErr) + } else { + for _, e := range opts.Exemplars { + latestTs := m.latestExemplarTs[ref] + if e.Ts < latestTs { + exErrs = append(exErrs, storage.ErrOutOfOrderExemplar) + continue + } + if e.Ts == latestTs { + // Similar to tsdb/head_append.go#appendExemplars, duplicate errors are not propagated. + continue + } + + m.latestExemplarTs[ref] = e.Ts + m.exemplars = append(m.exemplars, mockExemplar{ls, e.Labels, e.Ts, e.Value}) + } + } + + if len(exErrs) > 0 { + return storage.SeriesRef(ref), &storage.AppendPartialError{ExemplarErrors: exErrs} + } + return storage.SeriesRef(ref), nil } func (m *mockAppendable) Commit() error { @@ -1376,142 +1336,9 @@ func (m *mockAppendable) Rollback() error { m.samples = m.samples[:0] m.exemplars = m.exemplars[:0] m.histograms = m.histograms[:0] - m.metadata = m.metadata[:0] return nil } -func (m *mockAppendable) AppendExemplar(ref storage.SeriesRef, l labels.Labels, e exemplar.Exemplar) (storage.SeriesRef, error) { - if m.appendExemplarErr != nil { - return 0, m.appendExemplarErr - } - - latestTs := m.latestExemplar[uint64(ref)] - if e.Ts < latestTs { - return 0, storage.ErrOutOfOrderExemplar - } - if e.Ts == latestTs { - return 0, storage.ErrDuplicateExemplar - } - - m.latestExemplar[uint64(ref)] = e.Ts - m.exemplars = append(m.exemplars, mockExemplar{l, e.Labels, e.Ts, e.Value}) - return ref, nil -} - -func (m *mockAppendable) AppendHistogram(_ storage.SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { - if m.appendHistogramErr != nil { - return 0, m.appendHistogramErr - } - hash := l.Hash() - var latestTs int64 - if h != nil { - latestTs = m.latestHistogram[hash] - } else { - latestTs = m.latestFloatHist[hash] - } - if t < latestTs { - return 0, storage.ErrOutOfOrderSample - } - if t == latestTs { - return 0, storage.ErrDuplicateSampleForTimestamp - } - - if l.IsEmpty() { - return 0, tsdb.ErrInvalidSample - } - if _, hasDuplicates := l.HasDuplicateLabelNames(); hasDuplicates { - return 0, tsdb.ErrInvalidSample - } - - if h != nil { - m.latestHistogram[hash] = t - } else { - m.latestFloatHist[hash] = t - } - m.histograms = append(m.histograms, mockHistogram{l, t, h, fh}) - return storage.SeriesRef(hash), nil -} - -func (m *mockAppendable) AppendHistogramSTZeroSample(_ storage.SeriesRef, l labels.Labels, t, st int64, h *histogram.Histogram, _ *histogram.FloatHistogram) (storage.SeriesRef, error) { - if m.appendSTZeroSampleErr != nil { - return 0, m.appendSTZeroSampleErr - } - - // Created Timestamp can't be higher than the original sample's timestamp. - if st > t { - return 0, storage.ErrOutOfOrderSample - } - hash := l.Hash() - var latestTs int64 - if h != nil { - latestTs = m.latestHistogram[hash] - } else { - latestTs = m.latestFloatHist[hash] - } - if st < latestTs { - return 0, storage.ErrOutOfOrderSample - } - if st == latestTs { - return 0, storage.ErrDuplicateSampleForTimestamp - } - - if l.IsEmpty() { - return 0, tsdb.ErrInvalidSample - } - - if _, hasDuplicates := l.HasDuplicateLabelNames(); hasDuplicates { - return 0, tsdb.ErrInvalidSample - } - - if h != nil { - m.latestHistogram[hash] = st - m.histograms = append(m.histograms, mockHistogram{l, st, &histogram.Histogram{}, nil}) - } else { - m.latestFloatHist[hash] = st - m.histograms = append(m.histograms, mockHistogram{l, st, nil, &histogram.FloatHistogram{}}) - } - return storage.SeriesRef(hash), nil -} - -func (m *mockAppendable) UpdateMetadata(ref storage.SeriesRef, l labels.Labels, mp metadata.Metadata) (storage.SeriesRef, error) { - if m.updateMetadataErr != nil { - return 0, m.updateMetadataErr - } - - m.metadata = append(m.metadata, mockMetadata{l: l, m: mp}) - return ref, nil -} - -func (m *mockAppendable) AppendSTZeroSample(_ storage.SeriesRef, l labels.Labels, t, st int64) (storage.SeriesRef, error) { - if m.appendSTZeroSampleErr != nil { - return 0, m.appendSTZeroSampleErr - } - - // Created Timestamp can't be higher than the original sample's timestamp. - if st > t { - return 0, storage.ErrOutOfOrderSample - } - hash := l.Hash() - latestTs := m.latestSample[hash] - if st < latestTs { - return 0, storage.ErrOutOfOrderSample - } - if st == latestTs { - return 0, storage.ErrDuplicateSampleForTimestamp - } - - if l.IsEmpty() { - return 0, tsdb.ErrInvalidSample - } - if _, hasDuplicates := l.HasDuplicateLabelNames(); hasDuplicates { - return 0, tsdb.ErrInvalidSample - } - - m.latestSample[hash] = st - m.samples = append(m.samples, mockSample{l, st, 0}) - return storage.SeriesRef(hash), nil -} - var ( highSchemaHistogram = &histogram.Histogram{ Schema: 10, @@ -1553,7 +1380,7 @@ func TestHistogramsReduction(t *testing.T) { for _, protoMsg := range []remoteapi.WriteMessageType{remoteapi.WriteV1MessageType, remoteapi.WriteV2MessageType} { t.Run(string(protoMsg), func(t *testing.T) { appendable := &mockAppendable{} - handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{protoMsg}, false, false, false) + handler := NewWriteHandler(promslog.NewNopLogger(), nil, appendable, []remoteapi.WriteMessageType{protoMsg}, false) var ( err error diff --git a/storage/remote/write_test.go b/storage/remote/write_test.go index 2bf317465c..f4c8bd0b03 100644 --- a/storage/remote/write_test.go +++ b/storage/remote/write_test.go @@ -37,17 +37,16 @@ import ( common_config "github.com/prometheus/common/config" "github.com/prometheus/common/model" "github.com/prometheus/otlptranslator" - "github.com/stretchr/testify/require" - "go.opentelemetry.io/collector/pdata/pcommon" - "go.opentelemetry.io/collector/pdata/pmetric" - "go.opentelemetry.io/collector/pdata/pmetric/pmetricotlp" - "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/metadata" "github.com/prometheus/prometheus/model/relabel" "github.com/prometheus/prometheus/storage" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pmetric" + "go.opentelemetry.io/collector/pdata/pmetric/pmetricotlp" ) func testRemoteWriteConfig() *config.RemoteWriteConfig { @@ -385,87 +384,53 @@ func TestWriteStorageApplyConfig_PartialUpdate(t *testing.T) { require.NoError(t, s.Close()) } +// TODO(bwplotka): Move all the OTLP handler tests to `write_handler_test.go`. +// write.go and write_test.go are for sending (client side). + func TestOTLPWriteHandler(t *testing.T) { - timestamp := time.Now() - var zeroTime time.Time - exportRequest := generateOTLPWriteRequest(timestamp, zeroTime) + // Compile pieces of expectations that does not depend on translation or type and unit labels, for readability. + expectedBaseSamples := []mockSample{ + {m: metadata.Metadata{Type: model.MetricTypeCounter, Unit: "bytes", Help: "test-counter-description"}, v: 10.0}, + {m: metadata.Metadata{Type: model.MetricTypeGauge, Unit: "bytes", Help: "test-gauge-description"}, v: 10.0}, + {m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, v: 30.0}, + {m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, v: 12.0}, + {m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, v: 2.0}, + {m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, v: 4.0}, + {m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, v: 6.0}, + {m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, v: 8.0}, + {m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, v: 10.0}, + {m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, v: 12.0}, + {m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, v: 12.0}, + {m: metadata.Metadata{Type: model.MetricTypeGauge, Unit: "", Help: "Target metadata"}, v: 1}, + } + + ts := time.Now() + st := ts.Add(-1 * time.Millisecond) + exportRequest := generateOTLPWriteRequest(ts, st) for _, testCase := range []struct { name string otlpCfg config.OTLPConfig typeAndUnitLabels bool - expectedSamples []mockSample - expectedMetadata []mockMetadata + expectedSeries []labels.Labels }{ { name: "NoTranslation/NoTypeAndUnitLabels", otlpCfg: config.OTLPConfig{ TranslationStrategy: otlptranslator.NoTranslation, }, - expectedSamples: []mockSample{ - { - l: labels.FromStrings(model.MetricNameLabel, "test.counter", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - t: timestamp.UnixMilli(), - v: 10.0, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "target_info", "host.name", "test-host", "instance", "test-instance", "job", "test-service"), - t: timestamp.UnixMilli(), - v: 1, - }, - }, - expectedMetadata: []mockMetadata{ - { - l: labels.FromStrings(model.MetricNameLabel, "test.counter", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeCounter, Unit: "bytes", Help: "test-counter-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.gauge", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeGauge, Unit: "bytes", Help: "test-gauge-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_sum", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_count", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "0"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "1"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "2"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "3"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "4"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "5"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "+Inf"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.exponential.histogram", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-exponential-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "target_info", "host.name", "test-host", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeGauge, Unit: "", Help: "Target metadata"}, - }, + expectedSeries: []labels.Labels{ + labels.FromStrings(model.MetricNameLabel, "test.counter", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), + labels.FromStrings(model.MetricNameLabel, "test.gauge", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), + labels.FromStrings(model.MetricNameLabel, "test.histogram_sum", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), + labels.FromStrings(model.MetricNameLabel, "test.histogram_count", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), + labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "0"), + labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "1"), + labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "2"), + labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "3"), + labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "4"), + labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "5"), + labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "+Inf"), + labels.FromStrings(model.MetricNameLabel, "target_info", "host.name", "test-host", "instance", "test-instance", "job", "test-service"), }, }, { @@ -474,145 +439,39 @@ func TestOTLPWriteHandler(t *testing.T) { TranslationStrategy: otlptranslator.NoTranslation, }, typeAndUnitLabels: true, - expectedSamples: []mockSample{ - { - l: labels.FromStrings(model.MetricNameLabel, "test.counter", "__type__", "counter", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - t: timestamp.UnixMilli(), - v: 10.0, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "target_info", "host.name", "test-host", "instance", "test-instance", "job", "test-service"), - t: timestamp.UnixMilli(), - v: 1, - }, - }, - expectedMetadata: []mockMetadata{ - { - // Metadata labels follow series labels. - l: labels.FromStrings(model.MetricNameLabel, "test.counter", "__type__", "counter", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeCounter, Unit: "bytes", Help: "test-counter-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.gauge", "__type__", "gauge", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeGauge, Unit: "bytes", Help: "test-gauge-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_sum", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_count", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "0"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "1"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "2"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "3"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "4"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "5"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "+Inf"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.exponential.histogram", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-exponential-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "target_info", "host.name", "test-host", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeGauge, Unit: "", Help: "Target metadata"}, - }, + expectedSeries: []labels.Labels{ + labels.FromStrings(model.MetricNameLabel, "test.counter", "__type__", "counter", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), + labels.FromStrings(model.MetricNameLabel, "test.gauge", "__type__", "gauge", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), + labels.FromStrings(model.MetricNameLabel, "test.histogram_sum", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), + labels.FromStrings(model.MetricNameLabel, "test.histogram_count", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), + labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "0"), + labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "1"), + labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "2"), labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "3"), + labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "4"), + labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "5"), + labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "+Inf"), + labels.FromStrings(model.MetricNameLabel, "target_info", "host.name", "test-host", "instance", "test-instance", "job", "test-service"), }, }, + // For the following cases, skip type and unit cases, it has nothing todo with translation. { - name: "UnderscoreEscapingWithSuffixes/NoTypeAndUnitLabels", + name: "UnderscoreEscapingWithSuffixes", otlpCfg: config.OTLPConfig{ TranslationStrategy: otlptranslator.UnderscoreEscapingWithSuffixes, }, - expectedSamples: []mockSample{ - { - l: labels.FromStrings(model.MetricNameLabel, "test_counter_bytes_total", "foo_bar", "baz", "instance", "test-instance", "job", "test-service"), - t: timestamp.UnixMilli(), - v: 10.0, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "target_info", "host_name", "test-host", "instance", "test-instance", "job", "test-service"), - t: timestamp.UnixMilli(), - v: 1, - }, - }, - expectedMetadata: []mockMetadata{ - // All get _bytes unit suffix and counter also gets _total. - { - l: labels.FromStrings(model.MetricNameLabel, "test_counter_bytes_total", "foo_bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeCounter, Unit: "bytes", Help: "test-counter-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_gauge_bytes", "foo_bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeGauge, Unit: "bytes", Help: "test-gauge-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_histogram_bytes_sum", "foo_bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_histogram_bytes_count", "foo_bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_histogram_bytes_bucket", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "0"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_histogram_bytes_bucket", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "1"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_histogram_bytes_bucket", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "2"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_histogram_bytes_bucket", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "3"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_histogram_bytes_bucket", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "4"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_histogram_bytes_bucket", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "5"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_histogram_bytes_bucket", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "+Inf"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_exponential_histogram_bytes", "foo_bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-exponential-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "target_info", "host_name", "test-host", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeGauge, Unit: "", Help: "Target metadata"}, - }, + expectedSeries: []labels.Labels{ + labels.FromStrings(model.MetricNameLabel, "test_counter_bytes_total", "foo_bar", "baz", "instance", "test-instance", "job", "test-service"), + labels.FromStrings(model.MetricNameLabel, "test_gauge_bytes", "foo_bar", "baz", "instance", "test-instance", "job", "test-service"), + labels.FromStrings(model.MetricNameLabel, "test_histogram_bytes_sum", "foo_bar", "baz", "instance", "test-instance", "job", "test-service"), + labels.FromStrings(model.MetricNameLabel, "test_histogram_bytes_count", "foo_bar", "baz", "instance", "test-instance", "job", "test-service"), + labels.FromStrings(model.MetricNameLabel, "test_histogram_bytes_bucket", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "0"), + labels.FromStrings(model.MetricNameLabel, "test_histogram_bytes_bucket", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "1"), + labels.FromStrings(model.MetricNameLabel, "test_histogram_bytes_bucket", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "2"), + labels.FromStrings(model.MetricNameLabel, "test_histogram_bytes_bucket", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "3"), + labels.FromStrings(model.MetricNameLabel, "test_histogram_bytes_bucket", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "4"), + labels.FromStrings(model.MetricNameLabel, "test_histogram_bytes_bucket", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "5"), + labels.FromStrings(model.MetricNameLabel, "test_histogram_bytes_bucket", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "+Inf"), + labels.FromStrings(model.MetricNameLabel, "target_info", "host_name", "test-host", "instance", "test-instance", "job", "test-service"), }, }, { @@ -620,526 +479,68 @@ func TestOTLPWriteHandler(t *testing.T) { otlpCfg: config.OTLPConfig{ TranslationStrategy: otlptranslator.UnderscoreEscapingWithoutSuffixes, }, - expectedSamples: []mockSample{ - { - l: labels.FromStrings(model.MetricNameLabel, "test_counter", "foo_bar", "baz", "instance", "test-instance", "job", "test-service"), - t: timestamp.UnixMilli(), - v: 10.0, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "target_info", "host_name", "test-host", "instance", "test-instance", "job", "test-service"), - t: timestamp.UnixMilli(), - v: 1, - }, - }, - expectedMetadata: []mockMetadata{ - { - l: labels.FromStrings(model.MetricNameLabel, "test_counter", "foo_bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeCounter, Unit: "bytes", Help: "test-counter-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_gauge", "foo_bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeGauge, Unit: "bytes", Help: "test-gauge-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_histogram_sum", "foo_bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_histogram_count", "foo_bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_histogram_bucket", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "0"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_histogram_bucket", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "1"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_histogram_bucket", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "2"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_histogram_bucket", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "3"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_histogram_bucket", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "4"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_histogram_bucket", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "5"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_histogram_bucket", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "+Inf"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_exponential_histogram", "foo_bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-exponential-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "target_info", "host_name", "test-host", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeGauge, Unit: "", Help: "Target metadata"}, - }, + expectedSeries: []labels.Labels{ + labels.FromStrings(model.MetricNameLabel, "test_counter", "foo_bar", "baz", "instance", "test-instance", "job", "test-service"), + labels.FromStrings(model.MetricNameLabel, "test_gauge", "foo_bar", "baz", "instance", "test-instance", "job", "test-service"), + labels.FromStrings(model.MetricNameLabel, "test_histogram_sum", "foo_bar", "baz", "instance", "test-instance", "job", "test-service"), + labels.FromStrings(model.MetricNameLabel, "test_histogram_count", "foo_bar", "baz", "instance", "test-instance", "job", "test-service"), + labels.FromStrings(model.MetricNameLabel, "test_histogram_bucket", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "0"), + labels.FromStrings(model.MetricNameLabel, "test_histogram_bucket", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "1"), + labels.FromStrings(model.MetricNameLabel, "test_histogram_bucket", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "2"), + labels.FromStrings(model.MetricNameLabel, "test_histogram_bucket", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "3"), + labels.FromStrings(model.MetricNameLabel, "test_histogram_bucket", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "4"), + labels.FromStrings(model.MetricNameLabel, "test_histogram_bucket", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "5"), + labels.FromStrings(model.MetricNameLabel, "test_histogram_bucket", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "+Inf"), + labels.FromStrings(model.MetricNameLabel, "target_info", "host_name", "test-host", "instance", "test-instance", "job", "test-service"), }, }, { - name: "UnderscoreEscapingWithSuffixes/WithTypeAndUnitLabels", - otlpCfg: config.OTLPConfig{ - TranslationStrategy: otlptranslator.UnderscoreEscapingWithSuffixes, - }, - typeAndUnitLabels: true, - expectedSamples: []mockSample{ - { - l: labels.New(labels.Label{Name: "__name__", Value: "test_counter_bytes_total"}, - labels.Label{Name: "__type__", Value: "counter"}, - labels.Label{Name: "__unit__", Value: "bytes"}, - labels.Label{Name: "foo_bar", Value: "baz"}, - labels.Label{Name: "instance", Value: "test-instance"}, - labels.Label{Name: "job", Value: "test-service"}), - t: timestamp.UnixMilli(), - v: 10.0, - }, - { - l: labels.New( - labels.Label{Name: "__name__", Value: "target_info"}, - labels.Label{Name: "host_name", Value: "test-host"}, - labels.Label{Name: "instance", Value: "test-instance"}, - labels.Label{Name: "job", Value: "test-service"}, - ), - t: timestamp.UnixMilli(), - v: 1, - }, - }, - expectedMetadata: []mockMetadata{ - { - l: labels.FromStrings(model.MetricNameLabel, "test_counter_bytes_total", "__type__", "counter", "__unit__", "bytes", "foo_bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeCounter, Unit: "bytes", Help: "test-counter-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_gauge_bytes", "__type__", "gauge", "__unit__", "bytes", "foo_bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeGauge, Unit: "bytes", Help: "test-gauge-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_histogram_bytes_sum", "__type__", "histogram", "__unit__", "bytes", "foo_bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_histogram_bytes_count", "__type__", "histogram", "__unit__", "bytes", "foo_bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_histogram_bytes_bucket", "__type__", "histogram", "__unit__", "bytes", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "0"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_histogram_bytes_bucket", "__type__", "histogram", "__unit__", "bytes", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "1"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_histogram_bytes_bucket", "__type__", "histogram", "__unit__", "bytes", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "2"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_histogram_bytes_bucket", "__type__", "histogram", "__unit__", "bytes", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "3"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_histogram_bytes_bucket", "__type__", "histogram", "__unit__", "bytes", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "4"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_histogram_bytes_bucket", "__type__", "histogram", "__unit__", "bytes", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "5"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_histogram_bytes_bucket", "__type__", "histogram", "__unit__", "bytes", "foo_bar", "baz", "instance", "test-instance", "job", "test-service", "le", "+Inf"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test_exponential_histogram_bytes", "__type__", "histogram", "__unit__", "bytes", "foo_bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-exponential-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "target_info", "host_name", "test-host", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeGauge, Unit: "", Help: "Target metadata"}, - }, - }, - }, - { - name: "NoUTF8EscapingWithSuffixes/NoTypeAndUnitLabels", + name: "NoUTF8EscapingWithSuffixes", otlpCfg: config.OTLPConfig{ TranslationStrategy: otlptranslator.NoUTF8EscapingWithSuffixes, }, - expectedSamples: []mockSample{ - { - l: labels.FromStrings(model.MetricNameLabel, "test.counter_bytes_total", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - t: timestamp.UnixMilli(), - v: 10.0, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "target_info", "host.name", "test-host", "instance", "test-instance", "job", "test-service"), - t: timestamp.UnixMilli(), - v: 1, - }, - }, - expectedMetadata: []mockMetadata{ - // All get _bytes unit suffix and counter also gets _total. - { - l: labels.FromStrings(model.MetricNameLabel, "test.counter_bytes_total", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeCounter, Unit: "bytes", Help: "test-counter-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.gauge_bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeGauge, Unit: "bytes", Help: "test-gauge-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bytes_sum", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bytes_count", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bytes_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "0"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bytes_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "1"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bytes_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "2"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bytes_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "3"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bytes_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "4"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bytes_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "5"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bytes_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "+Inf"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.exponential.histogram_bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-exponential-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "target_info", "host.name", "test-host", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeGauge, Unit: "", Help: "Target metadata"}, - }, - }, - }, - { - name: "NoUTF8EscapingWithSuffixes/WithTypeAndUnitLabels", - otlpCfg: config.OTLPConfig{ - TranslationStrategy: otlptranslator.NoUTF8EscapingWithSuffixes, - }, - typeAndUnitLabels: true, - expectedSamples: []mockSample{ - { - l: labels.FromStrings(model.MetricNameLabel, "test.counter_bytes_total", "__type__", "counter", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - t: timestamp.UnixMilli(), - v: 10.0, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "target_info", "host.name", "test-host", "instance", "test-instance", "job", "test-service"), - t: timestamp.UnixMilli(), - v: 1, - }, - }, - expectedMetadata: []mockMetadata{ - // All get _bytes unit suffix and counter also gets _total. - { - l: labels.FromStrings(model.MetricNameLabel, "test.counter_bytes_total", "__type__", "counter", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeCounter, Unit: "bytes", Help: "test-counter-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.gauge_bytes", "__type__", "gauge", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeGauge, Unit: "bytes", Help: "test-gauge-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bytes_sum", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bytes_count", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bytes_bucket", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "0"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bytes_bucket", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "1"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bytes_bucket", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "2"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bytes_bucket", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "3"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bytes_bucket", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "4"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bytes_bucket", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "5"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bytes_bucket", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "+Inf"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.exponential.histogram_bytes", "__type__", "histogram", "__unit__", "bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeHistogram, Unit: "bytes", Help: "test-exponential-histogram-description"}, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "target_info", "host.name", "test-host", "instance", "test-instance", "job", "test-service"), - m: metadata.Metadata{Type: model.MetricTypeGauge, Unit: "", Help: "Target metadata"}, - }, + expectedSeries: []labels.Labels{ + labels.FromStrings(model.MetricNameLabel, "test.counter_bytes_total", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), + labels.FromStrings(model.MetricNameLabel, "test.gauge_bytes", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), + labels.FromStrings(model.MetricNameLabel, "test.histogram_bytes_sum", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), + labels.FromStrings(model.MetricNameLabel, "test.histogram_bytes_count", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), + labels.FromStrings(model.MetricNameLabel, "test.histogram_bytes_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "0"), + labels.FromStrings(model.MetricNameLabel, "test.histogram_bytes_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "1"), + labels.FromStrings(model.MetricNameLabel, "test.histogram_bytes_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "2"), + labels.FromStrings(model.MetricNameLabel, "test.histogram_bytes_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "3"), + labels.FromStrings(model.MetricNameLabel, "test.histogram_bytes_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "4"), + labels.FromStrings(model.MetricNameLabel, "test.histogram_bytes_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "5"), + labels.FromStrings(model.MetricNameLabel, "test.histogram_bytes_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "+Inf"), + labels.FromStrings(model.MetricNameLabel, "target_info", "host.name", "test-host", "instance", "test-instance", "job", "test-service"), }, }, } { t.Run(testCase.name, func(t *testing.T) { otlpOpts := OTLPOptions{ EnableTypeAndUnitLabels: testCase.typeAndUnitLabels, - AppendMetadata: true, } appendable := handleOTLP(t, exportRequest, testCase.otlpCfg, otlpOpts) - for _, sample := range testCase.expectedSamples { - requireContainsSample(t, appendable.samples, sample) - } - for _, meta := range testCase.expectedMetadata { - requireContainsMetadata(t, appendable.metadata, meta) + + expectedSamples := expectedBaseSamples + for i, l := range testCase.expectedSeries { + expectedSamples[i].l = l + expectedSamples[i].t = ts.UnixMilli() + expectedSamples[i].st = st.UnixMilli() + if l.Get(model.MetricNameLabel) == "target_info" { + expectedSamples[i].st = 0 // Target info is artificial and it does not have st (also gauge). + } } + requireEqual(t, expectedSamples, appendable.samples) + + // TODO: Test histogram sample? + require.Len(t, appendable.samples, 12) // 1 (counter) + 1 (gauge) + 1 (target_info) + 7 (hist_bucket) + 2 (hist_sum, hist_count) require.Len(t, appendable.histograms, 1) // 1 (exponential histogram) - require.Len(t, appendable.metadata, 13) // for each float and histogram sample require.Len(t, appendable.exemplars, 1) // 1 (exemplar) }) } } -// Check that start time is ingested if ingestSTZeroSample is enabled -// and the start time is actually set (non-zero). -func TestOTLPWriteHandler_StartTime(t *testing.T) { - timestamp := time.Now() - startTime := timestamp.Add(-1 * time.Millisecond) - var zeroTime time.Time - - expectedSamples := []mockSample{ - { - l: labels.FromStrings(model.MetricNameLabel, "test.counter", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - t: timestamp.UnixMilli(), - v: 10.0, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.gauge", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - t: timestamp.UnixMilli(), - v: 10.0, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_sum", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - t: timestamp.UnixMilli(), - v: 30.0, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_count", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - t: timestamp.UnixMilli(), - v: 12.0, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "0"), - t: timestamp.UnixMilli(), - v: 2.0, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "1"), - t: timestamp.UnixMilli(), - v: 4.0, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "2"), - t: timestamp.UnixMilli(), - v: 6.0, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "3"), - t: timestamp.UnixMilli(), - v: 8.0, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "4"), - t: timestamp.UnixMilli(), - v: 10.0, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "5"), - t: timestamp.UnixMilli(), - v: 12.0, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "test.histogram_bucket", "foo.bar", "baz", "instance", "test-instance", "job", "test-service", "le", "+Inf"), - t: timestamp.UnixMilli(), - v: 12.0, - }, - { - l: labels.FromStrings(model.MetricNameLabel, "target_info", "host.name", "test-host", "instance", "test-instance", "job", "test-service"), - t: timestamp.UnixMilli(), - v: 1.0, - }, - } - expectedHistograms := []mockHistogram{ - { - l: labels.FromStrings(model.MetricNameLabel, "test.exponential.histogram", "foo.bar", "baz", "instance", "test-instance", "job", "test-service"), - t: timestamp.UnixMilli(), - h: &histogram.Histogram{ - Schema: 2, - ZeroThreshold: 1e-128, - ZeroCount: 2, - Count: 10, - Sum: 30, - PositiveSpans: []histogram.Span{{Offset: 1, Length: 5}}, - PositiveBuckets: []int64{2, 0, 0, 0, 0}, - }, - }, - } - - expectedSamplesWithSTZero := make([]mockSample, 0, len(expectedSamples)*2-1) // All samples will get ST zero, except target_info. - for _, s := range expectedSamples { - if s.l.Get(model.MetricNameLabel) != "target_info" { - expectedSamplesWithSTZero = append(expectedSamplesWithSTZero, mockSample{ - l: s.l.Copy(), - t: startTime.UnixMilli(), - v: 0, - }) - } - expectedSamplesWithSTZero = append(expectedSamplesWithSTZero, s) - } - expectedHistogramsWithSTZero := make([]mockHistogram, 0, len(expectedHistograms)*2) - for _, s := range expectedHistograms { - if s.l.Get(model.MetricNameLabel) != "target_info" { - expectedHistogramsWithSTZero = append(expectedHistogramsWithSTZero, mockHistogram{ - l: s.l.Copy(), - t: startTime.UnixMilli(), - h: &histogram.Histogram{}, - }) - } - expectedHistogramsWithSTZero = append(expectedHistogramsWithSTZero, s) - } - - for _, testCase := range []struct { - name string - otlpOpts OTLPOptions - startTime time.Time - expectSTZero bool - expectedSamples []mockSample - expectedHistograms []mockHistogram - }{ - { - name: "IngestSTZero=false/startTime=0", - otlpOpts: OTLPOptions{ - IngestSTZeroSample: false, - }, - startTime: zeroTime, - expectedSamples: expectedSamples, - expectedHistograms: expectedHistograms, - }, - { - name: "IngestSTZero=true/startTime=0", - otlpOpts: OTLPOptions{ - IngestSTZeroSample: true, - }, - startTime: zeroTime, - expectedSamples: expectedSamples, - expectedHistograms: expectedHistograms, - }, - { - name: "IngestSTZero=false/startTime=ts-1ms", - otlpOpts: OTLPOptions{ - IngestSTZeroSample: false, - }, - startTime: startTime, - expectedSamples: expectedSamples, - expectedHistograms: expectedHistograms, - }, - { - name: "IngestSTZero=true/startTime=ts-1ms", - otlpOpts: OTLPOptions{ - IngestSTZeroSample: true, - }, - startTime: startTime, - expectedSamples: expectedSamplesWithSTZero, - expectedHistograms: expectedHistogramsWithSTZero, - }, - } { - t.Run(testCase.name, func(t *testing.T) { - exportRequest := generateOTLPWriteRequest(timestamp, testCase.startTime) - appendable := handleOTLP(t, exportRequest, config.OTLPConfig{ - TranslationStrategy: otlptranslator.NoTranslation, - }, testCase.otlpOpts) - for i, expect := range testCase.expectedSamples { - actual := appendable.samples[i] - require.True(t, labels.Equal(expect.l, actual.l), "sample labels,pos=%v", i) - require.Equal(t, expect.t, actual.t, "sample timestamp,pos=%v", i) - require.Equal(t, expect.v, actual.v, "sample value,pos=%v", i) - } - for i, expect := range testCase.expectedHistograms { - actual := appendable.histograms[i] - require.True(t, labels.Equal(expect.l, actual.l), "histogram labels,pos=%v", i) - require.Equal(t, expect.t, actual.t, "histogram timestamp,pos=%v", i) - require.True(t, expect.h.Equals(actual.h), "histogram value,pos=%v", i) - } - require.Len(t, appendable.samples, len(testCase.expectedSamples)) - require.Len(t, appendable.histograms, len(testCase.expectedHistograms)) - }) - } -} - -func requireContainsSample(t *testing.T, actual []mockSample, expected mockSample) { - t.Helper() - - for _, got := range actual { - if labels.Equal(expected.l, got.l) && expected.t == got.t && expected.v == got.v { - return - } - } - require.Fail(t, fmt.Sprintf("Sample not found: \n"+ - "expected: %v\n"+ - "actual : %v", expected, actual)) -} - -func requireContainsMetadata(t *testing.T, actual []mockMetadata, expected mockMetadata) { - t.Helper() - - for _, got := range actual { - if labels.Equal(expected.l, got.l) && expected.m.Type == got.m.Type && expected.m.Unit == got.m.Unit && expected.m.Help == got.m.Help { - return - } - } - require.Fail(t, fmt.Sprintf("Metadata not found: \n"+ - "expected: %v\n"+ - "actual : %v", expected, actual)) -} - func handleOTLP(t *testing.T, exportRequest pmetricotlp.ExportRequest, otlpCfg config.OTLPConfig, otlpOpts OTLPOptions) *mockAppendable { buf, err := exportRequest.MarshalProto() require.NoError(t, err) @@ -1164,7 +565,7 @@ func handleOTLP(t *testing.T, exportRequest pmetricotlp.ExportRequest, otlpCfg c return appendable } -func generateOTLPWriteRequest(timestamp, startTime time.Time) pmetricotlp.ExportRequest { +func generateOTLPWriteRequest(ts, startTime time.Time) pmetricotlp.ExportRequest { d := pmetric.NewMetrics() // Generate One Counter, One Gauge, One Histogram, One Exponential-Histogram @@ -1188,14 +589,14 @@ func generateOTLPWriteRequest(timestamp, startTime time.Time) pmetricotlp.Export counterMetric.Sum().SetIsMonotonic(true) counterDataPoint := counterMetric.Sum().DataPoints().AppendEmpty() - counterDataPoint.SetTimestamp(pcommon.NewTimestampFromTime(timestamp)) + counterDataPoint.SetTimestamp(pcommon.NewTimestampFromTime(ts)) counterDataPoint.SetStartTimestamp(pcommon.NewTimestampFromTime(startTime)) counterDataPoint.SetDoubleValue(10.0) counterDataPoint.Attributes().PutStr("foo.bar", "baz") counterExemplar := counterDataPoint.Exemplars().AppendEmpty() - counterExemplar.SetTimestamp(pcommon.NewTimestampFromTime(timestamp)) + counterExemplar.SetTimestamp(pcommon.NewTimestampFromTime(ts)) counterExemplar.SetDoubleValue(10.0) counterExemplar.SetSpanID(pcommon.SpanID{0, 1, 2, 3, 4, 5, 6, 7}) counterExemplar.SetTraceID(pcommon.TraceID{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}) @@ -1208,7 +609,7 @@ func generateOTLPWriteRequest(timestamp, startTime time.Time) pmetricotlp.Export gaugeMetric.SetEmptyGauge() gaugeDataPoint := gaugeMetric.Gauge().DataPoints().AppendEmpty() - gaugeDataPoint.SetTimestamp(pcommon.NewTimestampFromTime(timestamp)) + gaugeDataPoint.SetTimestamp(pcommon.NewTimestampFromTime(ts)) gaugeDataPoint.SetStartTimestamp(pcommon.NewTimestampFromTime(startTime)) gaugeDataPoint.SetDoubleValue(10.0) gaugeDataPoint.Attributes().PutStr("foo.bar", "baz") @@ -1222,7 +623,7 @@ func generateOTLPWriteRequest(timestamp, startTime time.Time) pmetricotlp.Export histogramMetric.Histogram().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) histogramDataPoint := histogramMetric.Histogram().DataPoints().AppendEmpty() - histogramDataPoint.SetTimestamp(pcommon.NewTimestampFromTime(timestamp)) + histogramDataPoint.SetTimestamp(pcommon.NewTimestampFromTime(ts)) histogramDataPoint.SetStartTimestamp(pcommon.NewTimestampFromTime(startTime)) histogramDataPoint.ExplicitBounds().FromRaw([]float64{0.0, 1.0, 2.0, 3.0, 4.0, 5.0}) histogramDataPoint.BucketCounts().FromRaw([]uint64{2, 2, 2, 2, 2, 2}) @@ -1239,7 +640,7 @@ func generateOTLPWriteRequest(timestamp, startTime time.Time) pmetricotlp.Export exponentialHistogramMetric.ExponentialHistogram().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) exponentialHistogramDataPoint := exponentialHistogramMetric.ExponentialHistogram().DataPoints().AppendEmpty() - exponentialHistogramDataPoint.SetTimestamp(pcommon.NewTimestampFromTime(timestamp)) + exponentialHistogramDataPoint.SetTimestamp(pcommon.NewTimestampFromTime(ts)) exponentialHistogramDataPoint.SetStartTimestamp(pcommon.NewTimestampFromTime(startTime)) exponentialHistogramDataPoint.SetScale(2.0) exponentialHistogramDataPoint.Positive().BucketCounts().FromRaw([]uint64{2, 2, 2, 2, 2}) @@ -1292,9 +693,9 @@ func TestOTLPDelta(t *testing.T) { } want := []mockSample{ - {t: milli(0), l: ls, v: 0}, // +0 - {t: milli(1), l: ls, v: 1}, // +1 - {t: milli(2), l: ls, v: 3}, // +2 + {t: milli(0), l: ls, m: metadata.Metadata{Type: model.MetricTypeGauge}, v: 0}, // +0 + {t: milli(1), l: ls, m: metadata.Metadata{Type: model.MetricTypeGauge}, v: 1}, // +1 + {t: milli(2), l: ls, m: metadata.Metadata{Type: model.MetricTypeGauge}, v: 3}, // +2 } if diff := cmp.Diff(want, appendable.samples, cmp.Exporter(func(reflect.Type) bool { return true })); diff != "" { t.Fatal(diff) @@ -1474,7 +875,7 @@ func BenchmarkOTLP(b *testing.B) { var total int // reqs is a [b.N]*http.Request, divided across the workers. - // deltatocumulative requires timestamps to be strictly in + // deltatocumulative requires tss to be strictly in // order on a per-series basis. to ensure this, each reqs[k] // contains samples of differently named series, sorted // strictly in time order @@ -1506,8 +907,8 @@ func BenchmarkOTLP(b *testing.B) { } log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn})) - mock := new(mockAppendable) - appendable := syncAppendable{Appendable: mock, lock: new(sync.Mutex)} + mock := &mockAppendable{} + appendable := syncAppendable{AppendableV2: mock, lock: new(sync.Mutex)} cfgfn := func() config.Config { return config.Config{OTLPConfig: config.DefaultOTLPConfig} } @@ -1588,28 +989,22 @@ func sampleCount(md pmetric.Metrics) int { type syncAppendable struct { lock sync.Locker - storage.Appendable + storage.AppendableV2 } type syncAppender struct { lock sync.Locker - storage.Appender + storage.AppenderV2 } -func (s syncAppendable) Appender(ctx context.Context) storage.Appender { - return syncAppender{Appender: s.Appendable.Appender(ctx), lock: s.lock} +func (s syncAppendable) AppenderV2(ctx context.Context) storage.AppenderV2 { + return syncAppender{AppenderV2: s.AppendableV2.AppenderV2(ctx), lock: s.lock} } -func (s syncAppender) Append(ref storage.SeriesRef, l labels.Labels, t int64, v float64) (storage.SeriesRef, error) { +func (s syncAppender) Append(ref storage.SeriesRef, ls labels.Labels, st, t int64, v float64, h *histogram.Histogram, fh *histogram.FloatHistogram, opts storage.AOptions) (storage.SeriesRef, error) { s.lock.Lock() defer s.lock.Unlock() - return s.Appender.Append(ref, l, t, v) -} - -func (s syncAppender) AppendHistogram(ref storage.SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, f *histogram.FloatHistogram) (storage.SeriesRef, error) { - s.lock.Lock() - defer s.lock.Unlock() - return s.Appender.AppendHistogram(ref, l, t, h, f) + return s.AppenderV2.Append(ref, ls, st, t, v, h, fh, opts) } func TestWriteStorage_CanRegisterMetricsAfterClosing(t *testing.T) { diff --git a/util/teststorage/appender.go b/util/teststorage/appender.go new file mode 100644 index 0000000000..faa0f317fa --- /dev/null +++ b/util/teststorage/appender.go @@ -0,0 +1,254 @@ +package teststorage + +import ( + "bytes" + "context" + "fmt" + "math" + "strings" + "sync" + + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/exemplar" + "github.com/prometheus/prometheus/model/histogram" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/model/metadata" + "github.com/prometheus/prometheus/storage" +) + +// Sample represents test, combined sample for mocking storage.AppenderV2. +type Sample struct { + MF string + L labels.Labels + M metadata.Metadata + ST, T int64 + V float64 + H *histogram.Histogram + FH *histogram.FloatHistogram + ES []exemplar.Exemplar +} + +func (s Sample) String() string { + b := bytes.Buffer{} + if s.M.Help != "" { + _, _ = fmt.Fprintf(&b, "HELP %s\n", s.M.Help) + } + if s.M.Type != model.MetricTypeUnknown && s.M.Type != "" { + _, _ = fmt.Fprintf(&b, "type@%s ", s.M.Type) + } + if s.M.Unit != "" { + _, _ = fmt.Fprintf(&b, "unit@%s ", s.M.Unit) + } + h := "" + if s.H != nil { + h = s.H.String() + } + + fh := "" + if s.FH != nil { + fh = s.FH.String() + } + _, _ = fmt.Fprintf(&b, "%s %v%v%v st@%v t@%v\n", s.L.String(), s.V, h, fh, s.ST, s.T) + return b.String() +} + +func (s Sample) exemplarsEqual(other []exemplar.Exemplar) bool { + if len(s.ES) != len(other) { + return false + } + for i := range s.ES { + if !s.ES[i].Equals(other[i]) { + return false + } + } + return true +} + +func (s Sample) Equal(other Sample) bool { + return strings.Compare(s.MF, other.MF) == 0 && + labels.Equal(s.L, other.L) && + s.M.Equals(other.M) && + s.ST == other.ST && + s.T == other.T && + math.Float64bits(s.V) == math.Float64bits(s.V) && // Compare Float64bits so NaN values which are exactly the same will compare equal. + s.H.Equals(other.H) && + s.FH.Equals(other.FH) && + s.exemplarsEqual(other.ES) +} + +// Appender is a storage.AppenderV2 mock. +// It allows: +// * recording all samples that were added through the appender. +// * optionally backed by another appender it writes samples through (Next). +// * optionally runs another appender before result recording e.g. to simulate chained validation (Prev) +// TODO(bwplotka): Move to storage/interface/mock or something? +type Appender struct { + Prev storage.AppendableV2 // Optional appender to run before the result collection. + Next storage.AppendableV2 // Optional appender to run after results are collected (e.g. TestStorage). + + AppendErr error // Inject appender error on every Append run. + AppendAllExemplarsError error // Inject storage.AppendPartialError for all exemplars. + CommitErr error // Inject commit error. + + mtx sync.Mutex // mutex for result writes and ResultSamplesGreaterThan read. + + // Recorded results. + PendingSamples []Sample + ResultSamples []Sample + RolledbackSamples []Sample +} + +func (a *Appender) ResultReset() { + a.PendingSamples = a.PendingSamples[:0] + a.ResultSamples = a.ResultSamples[:0] + a.RolledbackSamples = a.RolledbackSamples[:0] +} + +func (a *Appender) ResultSamplesGreaterThan(than int) bool { + a.mtx.Lock() + defer a.mtx.Unlock() + return len(a.ResultSamples) > than +} + +// ResultMetadata returns ResultSamples with samples only containing L and M. +// This is for compatibility with old tests that only focus on metadata. +// +// Deprecated: Rewrite tests to test metadata on ResultSamples instead. +func (a *Appender) ResultMetadata() []Sample { + var ret []Sample + for _, s := range a.ResultSamples { + ret = append(ret, Sample{L: s.L, M: s.M}) + } + return ret +} + +func (a *Appender) String() string { + var sb strings.Builder + sb.WriteString("committed:\n") + for _, s := range a.ResultSamples { + sb.WriteString("\n") + sb.WriteString(s.String()) + } + sb.WriteString("pending:\n") + for _, s := range a.PendingSamples { + sb.WriteString("\n") + sb.WriteString(s.String()) + } + sb.WriteString("rolledback:\n") + for _, s := range a.RolledbackSamples { + sb.WriteString("\n") + sb.WriteString(s.String()) + } + return sb.String() +} + +func NewAppender() *Appender { + return &Appender{} +} + +type appender struct { + prev storage.AppenderV2 + next storage.AppenderV2 + + *Appender +} + +func (a *Appender) AppenderV2(ctx context.Context) storage.AppenderV2 { + ret := &appender{Appender: a} + if a.Prev != nil { + ret.prev = a.Prev.AppenderV2(ctx) + } + if a.Next != nil { + ret.next = a.Next.AppenderV2(ctx) + } + return ret +} + +func (a *appender) Append(ref storage.SeriesRef, ls labels.Labels, st, t int64, v float64, h *histogram.Histogram, fh *histogram.FloatHistogram, opts storage.AOptions) (storage.SeriesRef, error) { + if a.Prev != nil { + if _, err := a.prev.Append(ref, ls, st, t, v, h, fh, opts); err != nil { + return 0, err + } + } + + if a.AppendErr != nil { + return 0, a.AppendErr + } + + a.mtx.Lock() + a.PendingSamples = append(a.PendingSamples, Sample{ + MF: opts.MetricFamilyName, + M: opts.Metadata, + L: ls, + ST: st, T: t, + V: v, H: h, FH: fh, + ES: opts.Exemplars, + }) + a.mtx.Unlock() + + var err error + if a.AppendAllExemplarsError != nil { + var exErrs []error + for range opts.Exemplars { + exErrs = append(exErrs, a.AppendAllExemplarsError) + } + if len(exErrs) > 0 { + err = &storage.AppendPartialError{ExemplarErrors: exErrs} + } + if ref == 0 { + // Use labels hash as a stand-in for unique series reference, to avoid having to track all series. + ref = storage.SeriesRef(ls.Hash()) + } + return ref, err + } + + if a.next != nil { + return a.next.Append(ref, ls, st, t, v, h, fh, opts) + } + + if ref == 0 { + // Use labels hash as a stand-in for unique series reference, to avoid having to track all series. + ref = storage.SeriesRef(ls.Hash()) + } + return ref, nil +} + +func (a *appender) Commit() error { + if a.Prev != nil { + if err := a.prev.Commit(); err != nil { + return err + } + } + + if a.CommitErr != nil { + return a.CommitErr + } + + a.mtx.Lock() + a.ResultSamples = append(a.ResultSamples, a.PendingSamples...) + a.PendingSamples = a.PendingSamples[:0] + a.mtx.Unlock() + + if a.next != nil { + return a.next.Commit() + } + return nil +} + +func (a *appender) Rollback() error { + if a.prev != nil { + if err := a.prev.Rollback(); err != nil { + return err + } + } + + a.mtx.Lock() + a.RolledbackSamples = append(a.RolledbackSamples, a.PendingSamples...) + a.PendingSamples = a.PendingSamples[:0] + a.mtx.Unlock() + + if a.next != nil { + return a.next.Rollback() + } + return nil +} diff --git a/util/teststorage/appender_test.go b/util/teststorage/appender_test.go new file mode 100644 index 0000000000..90eb0e796b --- /dev/null +++ b/util/teststorage/appender_test.go @@ -0,0 +1,41 @@ +package teststorage + +import ( + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/prometheus/prometheus/model/exemplar" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/model/metadata" + "github.com/prometheus/prometheus/util/testutil" + "github.com/stretchr/testify/require" +) + +func TestSample_RequireEqual(t *testing.T) { + a := []Sample{ + {}, + {L: labels.FromStrings("__name__", "test_metric_total"), M: metadata.Metadata{Type: "counter", Unit: "metric", Help: "some help text"}}, + {L: labels.FromStrings("__name__", "test_metric2", "foo", "bar"), M: metadata.Metadata{Type: "gauge", Unit: "", Help: "other help text"}, V: 123.123}, + {ES: []exemplar.Exemplar{{Labels: labels.FromStrings("__name__", "yolo")}}}, + } + testutil.RequireEqual(t, a, a) + + b1 := []Sample{ + {}, + {L: labels.FromStrings("__name__", "test_metric_total"), M: metadata.Metadata{Type: "counter", Unit: "metric", Help: "some help text"}}, + {L: labels.FromStrings("__name__", "test_metric2_diff", "foo", "bar"), M: metadata.Metadata{Type: "gauge", Unit: "", Help: "other help text"}, V: 123.123}, // test_metric2_diff is different. + {ES: []exemplar.Exemplar{{Labels: labels.FromStrings("__name__", "yolo")}}}, + } + requireNotEqual(t, a, b1) +} + +func requireNotEqual(t testing.TB, a, b any) { + t.Helper() + if !cmp.Equal(a, b, cmp.Comparer(labels.Equal)) { + return + } + require.Fail(t, fmt.Sprintf("Equal, but expected not: \n"+ + "a: %s\n"+ + "b: %s", a, b)) +} diff --git a/util/teststorage/storage.go b/util/teststorage/storage.go index e0a6f39be2..7d2978968c 100644 --- a/util/teststorage/storage.go +++ b/util/teststorage/storage.go @@ -21,9 +21,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/require" - "github.com/prometheus/prometheus/model/exemplar" - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/util/testutil" ) @@ -84,15 +81,3 @@ func (s TestStorage) Close() error { } return os.RemoveAll(s.dir) } - -func (s TestStorage) ExemplarAppender() storage.ExemplarAppender { - return s -} - -func (s TestStorage) ExemplarQueryable() storage.ExemplarQueryable { - return s.exemplarStorage -} - -func (s TestStorage) AppendExemplar(ref storage.SeriesRef, l labels.Labels, e exemplar.Exemplar) (storage.SeriesRef, error) { - return ref, s.exemplarStorage.AddExemplar(l, e) -} diff --git a/web/api/v1/api.go b/web/api/v1/api.go index 86c0461087..6a3f75d8c5 100644 --- a/web/api/v1/api.go +++ b/web/api/v1/api.go @@ -261,7 +261,7 @@ type API struct { func NewAPI( qe promql.QueryEngine, q storage.SampleAndChunkQueryable, - ap storage.Appendable, + ap storage.AppendableV2, eq storage.ExemplarQueryable, spsr func(context.Context) ScrapePoolsRetriever, tr func(context.Context) TargetRetriever, @@ -290,10 +290,8 @@ func NewAPI( rwEnabled bool, acceptRemoteWriteProtoMsgs remoteapi.MessageTypes, otlpEnabled, otlpDeltaToCumulative, otlpNativeDeltaIngestion bool, - stZeroIngestionEnabled bool, lookbackDelta time.Duration, enableTypeAndUnitLabels bool, - appendMetadata bool, overrideErrorCode OverrideErrorCode, ) *API { a := &API{ @@ -339,16 +337,14 @@ func NewAPI( } if rwEnabled { - a.remoteWriteHandler = remote.NewWriteHandler(logger, registerer, ap, acceptRemoteWriteProtoMsgs, stZeroIngestionEnabled, enableTypeAndUnitLabels, appendMetadata) + a.remoteWriteHandler = remote.NewWriteHandler(logger, registerer, ap, acceptRemoteWriteProtoMsgs, enableTypeAndUnitLabels) } if otlpEnabled { a.otlpWriteHandler = remote.NewOTLPWriteHandler(logger, registerer, ap, configFunc, remote.OTLPOptions{ ConvertDelta: otlpDeltaToCumulative, NativeDelta: otlpNativeDeltaIngestion, LookbackDelta: lookbackDelta, - IngestSTZeroSample: stZeroIngestionEnabled, EnableTypeAndUnitLabels: enableTypeAndUnitLabels, - AppendMetadata: appendMetadata, }) }