From bd725fd6b8a27aef969ed97c28ced358f2f3bc99 Mon Sep 17 00:00:00 2001 From: machine424 Date: Wed, 20 Aug 2025 11:53:24 +0200 Subject: [PATCH 1/2] test(notifier): add a test showing an alert mutation bug between alertmanager_config (alertmanagersets) The alert_relabel_configs should only apply to the corresponding alertmanagerset Signed-off-by: machine424 --- notifier/manager_test.go | 85 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/notifier/manager_test.go b/notifier/manager_test.go index 6f8f687f9a..09ca49e5c7 100644 --- a/notifier/manager_test.go +++ b/notifier/manager_test.go @@ -1132,3 +1132,88 @@ alerting: require.NoError(t, n.ApplyConfig(cfg)) require.Empty(t, n.Alertmanagers()) } + +// TestAlerstRelabelingIsIsolated ensures that a mutation alerts relabeling in an +// alertmanagerSet doesn't affect others. +// See https://github.com/prometheus/prometheus/pull/17063. +func TestAlerstRelabelingIsIsolated(t *testing.T) { + var ( + errc = make(chan error, 1) + expected1 = make([]*Alert, 0) + expected2 = make([]*Alert, 0) + + status1, status2 atomic.Int32 + ) + status1.Store(int32(http.StatusOK)) + status2.Store(int32(http.StatusOK)) + + server1 := newTestHTTPServerBuilder(&expected1, errc, "", "", &status1) + server2 := newTestHTTPServerBuilder(&expected2, errc, "", "", &status2) + + defer server1.Close() + defer server2.Close() + + h := NewManager(&Options{}, model.UTF8Validation, nil) + h.alertmanagers = make(map[string]*alertmanagerSet) + + am1Cfg := config.DefaultAlertmanagerConfig + am1Cfg.Timeout = model.Duration(time.Second) + am1Cfg.AlertRelabelConfigs = []*relabel.Config{ + { + SourceLabels: model.LabelNames{"alertname"}, + Regex: relabel.MustNewRegexp("(.*)"), + TargetLabel: "parasite", + Action: relabel.Replace, + Replacement: "yes", + NameValidationScheme: model.UTF8Validation, + }, + } + + am2Cfg := config.DefaultAlertmanagerConfig + am2Cfg.Timeout = model.Duration(time.Second) + + h.alertmanagers = map[string]*alertmanagerSet{ + "am1": { + ams: []alertmanager{ + alertmanagerMock{ + urlf: func() string { return server1.URL }, + }, + }, + cfg: &am1Cfg, + }, + "am2": { + ams: []alertmanager{ + alertmanagerMock{ + urlf: func() string { return server2.URL }, + }, + }, + cfg: &am2Cfg, + }, + } + + testAlert := &Alert{ + Labels: labels.FromStrings("alertname", "test"), + } + h.queue = []*Alert{testAlert} + + expected1 = append(expected1, &Alert{ + Labels: labels.FromStrings("alertname", "test", "parasite", "yes"), + }) + + // am2 shouldn't get the parasite label. + expected2 = append(expected2, &Alert{ + Labels: labels.FromStrings("alertname", "test"), + }) + + checkNoErr := func() { + t.Helper() + select { + case err := <-errc: + require.NoError(t, err) + default: + } + } + + require.True(t, h.sendAll(h.queue...)) + checkNoErr() +} From 8f79470ca91f49f1492e8a92527251d461cc65ae Mon Sep 17 00:00:00 2001 From: machine424 Date: Mon, 25 Aug 2025 16:32:34 +0200 Subject: [PATCH 2/2] fix(notifier): create a new alert when relabeling alters labels Signed-off-by: machine424 --- notifier/alert.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/notifier/alert.go b/notifier/alert.go index 88245c9a7f..83e7a97fe0 100644 --- a/notifier/alert.go +++ b/notifier/alert.go @@ -84,7 +84,18 @@ func relabelAlerts(relabelConfigs []*relabel.Config, externalLabels labels.Label if !keep { continue } - a.Labels = lb.Labels() + + // If relabeling has altered the labels, create a new Alert to preserve immutability. + if !labels.Equal(a.Labels, lb.Labels()) { + a = &Alert{ + Labels: lb.Labels(), + Annotations: a.Annotations, + StartsAt: a.StartsAt, + EndsAt: a.EndsAt, + GeneratorURL: a.GeneratorURL, + } + } + relabeledAlerts = append(relabeledAlerts, a) } return relabeledAlerts