From bf7b83059c2b262abf881493064e53b6016540f1 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 23 Dec 2025 11:56:39 +0000 Subject: [PATCH 1/5] Prepare release candidate 3.9-rc.0 (#17716) Signed-off-by: Bryan Boreham --- CHANGELOG.md | 38 ++++++++++++++++++-- VERSION | 2 +- web/ui/mantine-ui/package.json | 4 +-- web/ui/module/codemirror-promql/package.json | 4 +-- web/ui/module/lezer-promql/package.json | 2 +- web/ui/package-lock.json | 14 ++++---- web/ui/package.json | 2 +- 7 files changed, 50 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 976be5f52f..05c9b71b0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,42 @@ # Changelog -## main / unreleased +## 3.9.0-rc.0 / 2025-12-18 -* [BUGFIX] TSDB: Register `prometheus_tsdb_sample_ooo_delta` metric properly. #17477 +- [CHANGE] Native Histograms are no longer experimental! Make the `native-histogram` feature flag a no-op. Use `scrape_native_histograms` config option instead. #17528 +- [CHANGE] API: Add maximum limit of 10,000 sets of statistics to TSDB status endpoint. #17647 +- [FEATURE] API: Add /api/v1/features for clients to understand which features are supported. #17427 +- [FEATURE] Promtool: Add `start_timestamp` field for unit tests. #17636 +- [FEATURE] Promtool: Add `--format seriesjson` option to `tsdb dump` to output just series labels in JSON format. #13409 +- [FEATURE] Add `--storage.tsdb.delay-compact-file.path` flag for better interoperability with Thanos. #17435 +- [FEATURE] UI: Add an option on the query drop-down menu to duplicate that query panel. #17714 +- [ENHANCEMENT]: TSDB: add flag `--storage.tsdb.block-reload-interval` to configure TSDB Block Reload Interval. #16728 +- [ENHANCEMENT] UI: Add graph option to start the chart's Y axis at zero. #17565 +- [ENHANCEMENT] Scraping: Classic protobuf format no longer requires the unit in the metric name. #16834 +- [ENHANCEMENT] PromQL, Rules, SD, Scraping: Add native histograms to complement existing summaries. #17374 +- [ENHANCEMENT] Notifications: Add a histogram `prometheus_notifications_latency_histogram_seconds` to complement the existing summary. #16637 +- [ENHANCEMENT] Remote-write: Add custom scope support for AzureAD authentication. #17483 +- [ENHANCEMENT] SD: add a `config` label with job name for most `prometheus_sd_refresh` metrics. #17138 +- [ENHANCEMENT] TSDB: New histogram `prometheus_tsdb_sample_ooo_delta`, the distribution of out-of-order samples in seconds. Collected for all samples, accepted or not. #17477 +- [ENHANCEMENT] Remote-read: Validate histograms received via remote-read. #17561 +- [PERF] TSDB: Small optimizations to postings index. #17439 +- [PERF] Scraping: Speed up relabelling of series. #17530 +- [PERF] PromQL: Small optimisations in binary operators. #17524, #17519. +- [BUGFIX] UI: PromQL autocomplete now shows the correct type and HELP text for OpenMetrics counters whose samples end in `_total`. #17682 +- [BUGFIX] UI: Fixed codemirror-promql incorrectly showing label completion suggestions after the closing curly brace of a vector selector. #17602 +- [BUGFIX] UI: Query editor no longer suggests a duration unit if one is already present after a number. #17605 +- [BUGFIX] PromQL: Fix some "vector cannot contain metrics with the same labelset" errors when experimental delayed name removal is enabled. #17678 +- [BUGFIX] PromQL: Fix possible corruption of PromQL text if the query had an empty `ignoring()` and non-empty grouping. #17643 +- [BUGFIX] PromQL: Fix resets/changes to return empty results for anchored selectors when all samples are outside the range. #17479 +- [BUGFIX] PromQL: Check more consistently for many-to-one matching in filter binary operators. #17668 +- [BUGFIX] PromQL: Fix collision in unary negation with non-overlapping series. #17708 +- [BUGFIX] PromQL: Fix collision in label_join and label_replace with non-overlapping series. #17703 +- [BUGFIX] PromQL: Fix bug with inconsistent results for queries with OR expression when experimental delayed name removal is enabled. #17161 +- [BUGFIX] PromQL: Ensure that `rate`/`increase`/`delta` of histograms results in a gauge histogram. #17608 +- [BUGFIX] PromQL: Do not panic while iterating over invalid histograms. #17559 +- [BUGFIX] TSDB: Reject chunk files whose encoded chunk length overflows int. #17533 +- [BUGFIX] TSDB: Do not panic during resolution reduction of invalid histograms. #17561 +- [BUGFIX] Remote-write Receive: Avoid duplicate labels when experimental type-and-unit-label feature is enabled. #17546 +- [BUGFIX] OTLP Receiver: Only write metadata to disk when experimental metadata-wal-records feature is enabled. #17472 ## 3.8.1 / 2025-12-16 diff --git a/VERSION b/VERSION index f280719674..44fc2364a9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.8.1 +3.9.0-rc.0 diff --git a/web/ui/mantine-ui/package.json b/web/ui/mantine-ui/package.json index baf47d6f6b..7958d5db91 100644 --- a/web/ui/mantine-ui/package.json +++ b/web/ui/mantine-ui/package.json @@ -1,7 +1,7 @@ { "name": "@prometheus-io/mantine-ui", "private": true, - "version": "0.308.1", + "version": "0.309.0-rc.0", "type": "module", "scripts": { "start": "vite", @@ -28,7 +28,7 @@ "@microsoft/fetch-event-source": "^2.0.1", "@nexucis/fuzzy": "^0.5.1", "@nexucis/kvsearch": "^0.9.1", - "@prometheus-io/codemirror-promql": "0.308.1", + "@prometheus-io/codemirror-promql": "0.309.0-rc.0", "@reduxjs/toolkit": "^2.10.1", "@tabler/icons-react": "^3.35.0", "@tanstack/react-query": "^5.90.7", diff --git a/web/ui/module/codemirror-promql/package.json b/web/ui/module/codemirror-promql/package.json index 5f632320bd..6ad2116497 100644 --- a/web/ui/module/codemirror-promql/package.json +++ b/web/ui/module/codemirror-promql/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/codemirror-promql", - "version": "0.308.1", + "version": "0.309.0-rc.0", "description": "a CodeMirror mode for the PromQL language", "types": "dist/esm/index.d.ts", "module": "dist/esm/index.js", @@ -29,7 +29,7 @@ }, "homepage": "https://github.com/prometheus/prometheus/blob/main/web/ui/module/codemirror-promql/README.md", "dependencies": { - "@prometheus-io/lezer-promql": "0.308.1", + "@prometheus-io/lezer-promql": "0.309.0-rc.0", "lru-cache": "^11.2.2" }, "devDependencies": { diff --git a/web/ui/module/lezer-promql/package.json b/web/ui/module/lezer-promql/package.json index 85cc4c50ed..d83e1a6488 100644 --- a/web/ui/module/lezer-promql/package.json +++ b/web/ui/module/lezer-promql/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/lezer-promql", - "version": "0.308.1", + "version": "0.309.0-rc.0", "description": "lezer-based PromQL grammar", "main": "dist/index.cjs", "type": "module", diff --git a/web/ui/package-lock.json b/web/ui/package-lock.json index 883ee7aaee..23ae580c20 100644 --- a/web/ui/package-lock.json +++ b/web/ui/package-lock.json @@ -1,12 +1,12 @@ { "name": "prometheus-io", - "version": "0.308.1", + "version": "0.309.0-rc.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "prometheus-io", - "version": "0.308.1", + "version": "0.309.0-rc.0", "workspaces": [ "mantine-ui", "module/*" @@ -24,7 +24,7 @@ }, "mantine-ui": { "name": "@prometheus-io/mantine-ui", - "version": "0.308.0", + "version": "0.309.0-rc.0", "dependencies": { "@codemirror/autocomplete": "^6.19.1", "@codemirror/language": "^6.11.3", @@ -42,7 +42,7 @@ "@microsoft/fetch-event-source": "^2.0.1", "@nexucis/fuzzy": "^0.5.1", "@nexucis/kvsearch": "^0.9.1", - "@prometheus-io/codemirror-promql": "0.308.0", + "@prometheus-io/codemirror-promql": "0.309.0-rc.0", "@reduxjs/toolkit": "^2.10.1", "@tabler/icons-react": "^3.35.0", "@tanstack/react-query": "^5.90.7", @@ -88,10 +88,10 @@ }, "module/codemirror-promql": { "name": "@prometheus-io/codemirror-promql", - "version": "0.308.0", + "version": "0.309.0-rc.0", "license": "Apache-2.0", "dependencies": { - "@prometheus-io/lezer-promql": "0.308.0", + "@prometheus-io/lezer-promql": "0.309.0-rc.0", "lru-cache": "^11.2.2" }, "devDependencies": { @@ -121,7 +121,7 @@ }, "module/lezer-promql": { "name": "@prometheus-io/lezer-promql", - "version": "0.308.0", + "version": "0.309.0-rc.0", "license": "Apache-2.0", "devDependencies": { "@lezer/generator": "^1.8.0", diff --git a/web/ui/package.json b/web/ui/package.json index 44d0b52ce0..dd7d25628a 100644 --- a/web/ui/package.json +++ b/web/ui/package.json @@ -1,7 +1,7 @@ { "name": "prometheus-io", "description": "Monorepo for the Prometheus UI", - "version": "0.308.1", + "version": "0.309.0-rc.0", "private": true, "scripts": { "build": "bash build_ui.sh --all", From cd875bd8c9211d7606981223d59ab3adf73432f2 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 6 Jan 2026 16:30:06 +0000 Subject: [PATCH 2/5] Cut release 3.9.0 (#17796) Signed-off-by: Bryan Boreham --- CHANGELOG.md | 2 +- VERSION | 2 +- web/ui/mantine-ui/package.json | 4 ++-- web/ui/module/codemirror-promql/package.json | 4 ++-- web/ui/module/lezer-promql/package.json | 2 +- web/ui/package-lock.json | 14 +++++++------- web/ui/package.json | 2 +- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05c9b71b0f..6113dd0156 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 3.9.0-rc.0 / 2025-12-18 +## 3.9.0 / 2026-01-06 - [CHANGE] Native Histograms are no longer experimental! Make the `native-histogram` feature flag a no-op. Use `scrape_native_histograms` config option instead. #17528 - [CHANGE] API: Add maximum limit of 10,000 sets of statistics to TSDB status endpoint. #17647 diff --git a/VERSION b/VERSION index 44fc2364a9..a5c4c76339 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.9.0-rc.0 +3.9.0 diff --git a/web/ui/mantine-ui/package.json b/web/ui/mantine-ui/package.json index 7958d5db91..3ee4c6c48c 100644 --- a/web/ui/mantine-ui/package.json +++ b/web/ui/mantine-ui/package.json @@ -1,7 +1,7 @@ { "name": "@prometheus-io/mantine-ui", "private": true, - "version": "0.309.0-rc.0", + "version": "0.309.0", "type": "module", "scripts": { "start": "vite", @@ -28,7 +28,7 @@ "@microsoft/fetch-event-source": "^2.0.1", "@nexucis/fuzzy": "^0.5.1", "@nexucis/kvsearch": "^0.9.1", - "@prometheus-io/codemirror-promql": "0.309.0-rc.0", + "@prometheus-io/codemirror-promql": "0.309.0", "@reduxjs/toolkit": "^2.10.1", "@tabler/icons-react": "^3.35.0", "@tanstack/react-query": "^5.90.7", diff --git a/web/ui/module/codemirror-promql/package.json b/web/ui/module/codemirror-promql/package.json index 6ad2116497..227dc67ed6 100644 --- a/web/ui/module/codemirror-promql/package.json +++ b/web/ui/module/codemirror-promql/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/codemirror-promql", - "version": "0.309.0-rc.0", + "version": "0.309.0", "description": "a CodeMirror mode for the PromQL language", "types": "dist/esm/index.d.ts", "module": "dist/esm/index.js", @@ -29,7 +29,7 @@ }, "homepage": "https://github.com/prometheus/prometheus/blob/main/web/ui/module/codemirror-promql/README.md", "dependencies": { - "@prometheus-io/lezer-promql": "0.309.0-rc.0", + "@prometheus-io/lezer-promql": "0.309.0", "lru-cache": "^11.2.2" }, "devDependencies": { diff --git a/web/ui/module/lezer-promql/package.json b/web/ui/module/lezer-promql/package.json index d83e1a6488..e1cf0ad67b 100644 --- a/web/ui/module/lezer-promql/package.json +++ b/web/ui/module/lezer-promql/package.json @@ -1,6 +1,6 @@ { "name": "@prometheus-io/lezer-promql", - "version": "0.309.0-rc.0", + "version": "0.309.0", "description": "lezer-based PromQL grammar", "main": "dist/index.cjs", "type": "module", diff --git a/web/ui/package-lock.json b/web/ui/package-lock.json index 23ae580c20..c52491732d 100644 --- a/web/ui/package-lock.json +++ b/web/ui/package-lock.json @@ -1,12 +1,12 @@ { "name": "prometheus-io", - "version": "0.309.0-rc.0", + "version": "0.309.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "prometheus-io", - "version": "0.309.0-rc.0", + "version": "0.309.0", "workspaces": [ "mantine-ui", "module/*" @@ -24,7 +24,7 @@ }, "mantine-ui": { "name": "@prometheus-io/mantine-ui", - "version": "0.309.0-rc.0", + "version": "0.309.0", "dependencies": { "@codemirror/autocomplete": "^6.19.1", "@codemirror/language": "^6.11.3", @@ -42,7 +42,7 @@ "@microsoft/fetch-event-source": "^2.0.1", "@nexucis/fuzzy": "^0.5.1", "@nexucis/kvsearch": "^0.9.1", - "@prometheus-io/codemirror-promql": "0.309.0-rc.0", + "@prometheus-io/codemirror-promql": "0.309.0", "@reduxjs/toolkit": "^2.10.1", "@tabler/icons-react": "^3.35.0", "@tanstack/react-query": "^5.90.7", @@ -88,10 +88,10 @@ }, "module/codemirror-promql": { "name": "@prometheus-io/codemirror-promql", - "version": "0.309.0-rc.0", + "version": "0.309.0", "license": "Apache-2.0", "dependencies": { - "@prometheus-io/lezer-promql": "0.309.0-rc.0", + "@prometheus-io/lezer-promql": "0.309.0", "lru-cache": "^11.2.2" }, "devDependencies": { @@ -121,7 +121,7 @@ }, "module/lezer-promql": { "name": "@prometheus-io/lezer-promql", - "version": "0.309.0-rc.0", + "version": "0.309.0", "license": "Apache-2.0", "devDependencies": { "@lezer/generator": "^1.8.0", diff --git a/web/ui/package.json b/web/ui/package.json index dd7d25628a..0f054c34a7 100644 --- a/web/ui/package.json +++ b/web/ui/package.json @@ -1,7 +1,7 @@ { "name": "prometheus-io", "description": "Monorepo for the Prometheus UI", - "version": "0.309.0-rc.0", + "version": "0.309.0", "private": true, "scripts": { "build": "bash build_ui.sh --all", From f1719fa1d4e56303f608031fb809c6e8b7b945b8 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 7 Jan 2026 14:01:02 +0000 Subject: [PATCH 3/5] [BUGFIX] Agent: fix crash from invalid type in pool (#17802) We have separate pools for Appender and AppenderV2 objects, and must not put another kind of object into them. Signed-off-by: Bryan Boreham --- tsdb/agent/db.go | 16 ++++++++++++---- tsdb/agent/db_append_v2.go | 10 ++++++++++ tsdb/agent/db_append_v2_test.go | 4 ++++ tsdb/agent/db_test.go | 3 +++ 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/tsdb/agent/db.go b/tsdb/agent/db.go index a0f7a93b6d..3e9d7ed714 100644 --- a/tsdb/agent/db.go +++ b/tsdb/agent/db.go @@ -1124,13 +1124,22 @@ func (a *appender) AppendSTZeroSample(ref storage.SeriesRef, l labels.Labels, t, } // Commit submits the collected samples and purges the batch. -func (a *appenderBase) Commit() error { +func (a *appender) Commit() error { + defer a.appenderPool.Put(a) + return a.commit() +} + +func (a *appender) Rollback() error { + defer a.appenderPool.Put(a) + return a.rollback() +} + +func (a *appenderBase) commit() error { if err := a.log(); err != nil { return err } a.clearData() - a.appenderPool.Put(a) if a.writeNotified != nil { a.writeNotified.Notify() @@ -1244,7 +1253,7 @@ func (a *appenderBase) clearData() { a.floatHistogramSeries = a.floatHistogramSeries[:0] } -func (a *appenderBase) Rollback() error { +func (a *appenderBase) rollback() error { // Series are created in-memory regardless of rollback. This means we must // log them to the WAL, otherwise subsequent commits may reference a series // which was never written to the WAL. @@ -1253,7 +1262,6 @@ func (a *appenderBase) Rollback() error { } a.clearData() - a.appenderPool.Put(a) return nil } diff --git a/tsdb/agent/db_append_v2.go b/tsdb/agent/db_append_v2.go index f356a4feae..bb2601e1e3 100644 --- a/tsdb/agent/db_append_v2.go +++ b/tsdb/agent/db_append_v2.go @@ -127,6 +127,16 @@ func (a *appenderV2) Append(ref storage.SeriesRef, ls labels.Labels, st, t int64 return storage.SeriesRef(s.ref), partialErr } +func (a *appenderV2) Commit() error { + defer a.appenderV2Pool.Put(a) + return a.commit() +} + +func (a *appenderV2) Rollback() error { + defer a.appenderV2Pool.Put(a) + return a.rollback() +} + func (a *appenderV2) appendExemplars(s *memSeries, exemplar []exemplar.Exemplar) error { var errs []error for _, e := range exemplar { diff --git a/tsdb/agent/db_append_v2_test.go b/tsdb/agent/db_append_v2_test.go index 6a85e93c35..3e10a1163b 100644 --- a/tsdb/agent/db_append_v2_test.go +++ b/tsdb/agent/db_append_v2_test.go @@ -224,6 +224,10 @@ func TestCommit_AppendV2(t *testing.T) { require.Equal(t, numSeries*numDatapoints, walExemplarsCount, "unexpected number of exemplars") require.Equal(t, numSeries*numHistograms*2, walHistogramCount, "unexpected number of histograms") require.Equal(t, numSeries*numHistograms*2, walFloatHistogramCount, "unexpected number of float histograms") + + // Check that we can still create both kinds of Appender - see https://github.com/prometheus/prometheus/issues/17800. + _ = s.Appender(context.TODO()) + _ = s.AppenderV2(context.TODO()) } func TestRollback_AppendV2(t *testing.T) { diff --git a/tsdb/agent/db_test.go b/tsdb/agent/db_test.go index 94e84fa2eb..d2e005c175 100644 --- a/tsdb/agent/db_test.go +++ b/tsdb/agent/db_test.go @@ -259,6 +259,9 @@ func TestCommit(t *testing.T) { require.Equal(t, numSeries*numDatapoints, walExemplarsCount, "unexpected number of exemplars") require.Equal(t, numSeries*numHistograms*2, walHistogramCount, "unexpected number of histograms") require.Equal(t, numSeries*numHistograms*2, walFloatHistogramCount, "unexpected number of float histograms") + + // Check that we can get another appender after this - see https://github.com/prometheus/prometheus/issues/17800. + _ = s.Appender(context.TODO()) } func TestRollback(t *testing.T) { From ae711852559722dbad0797195a89fd39ee0d2324 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 7 Jan 2026 14:56:10 +0000 Subject: [PATCH 4/5] Scraping: add a test for relabel with keep and drop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Marvin Rösch Signed-off-by: Bryan Boreham --- scrape/scrape_test.go | 44 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/scrape/scrape_test.go b/scrape/scrape_test.go index eab1499158..270d1909dd 100644 --- a/scrape/scrape_test.go +++ b/scrape/scrape_test.go @@ -5982,3 +5982,47 @@ func TestScrapeLoopDisableStalenessMarkerInjection(t *testing.T) { } } } + +func TestDropsSeriesFromRelabeling(t *testing.T) { + s := teststorage.New(t) + defer s.Close() + ctx := t.Context() + + target := &Target{} + relabelConfig := []*relabel.Config{ + { + SourceLabels: model.LabelNames{"__name__"}, + Regex: relabel.MustNewRegexp(".*_total$"), + Action: relabel.Keep, + }, + { + SourceLabels: model.LabelNames{"__name__"}, + Regex: relabel.MustNewRegexp("test_metric_2_total$"), + Action: relabel.Drop, + }, + } + metricsText := []byte(` +# HELP test_metric_1_total This is a counter +# TYPE test_metric_1_total counter +test_metric_1_total 123 +# HELP test_metric_2_total This is a counter +# TYPE test_metric_2_total counter +test_metric_2_total 234 +# HELP disk_usage_bytes This is a gauge +# TYPE disk_usage_bytes gauge +disk_usage_bytes 456 +`) + + sl := newBasicScrapeLoop(t, ctx, &testScraper{}, s.Appender, 0) + sl.sampleMutator = func(l labels.Labels) labels.Labels { + return mutateSampleLabels(l, target, true, relabelConfig) + } + + slApp := sl.appender(ctx) + total, added, seriesAdded, err := sl.append(slApp, metricsText, "text/plain", time.Time{}) + require.NoError(t, err) + require.NoError(t, slApp.Rollback()) + require.Equal(t, 3, total) + require.Equal(t, 1, added) + require.Equal(t, 1, seriesAdded) +} From 66c8e31956777630949b1be5009d1a7e6de83921 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marvin=20R=C3=B6sch?= Date: Wed, 7 Jan 2026 15:27:01 +0100 Subject: [PATCH 5/5] [BUGFIX] Scraping: drop sample if relabeling config says so MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marvin Rösch --- scrape/scrape.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrape/scrape.go b/scrape/scrape.go index b653873bad..a8cd15d30c 100644 --- a/scrape/scrape.go +++ b/scrape/scrape.go @@ -716,7 +716,9 @@ func mutateSampleLabels(lset labels.Labels, target *Target, honor bool, rc []*re } } - relabel.ProcessBuilder(lb, rc...) + if keep := relabel.ProcessBuilder(lb, rc...); !keep { + return labels.EmptyLabels() + } return lb.Labels() }