From caa71fb3c74f2fb10bcdf2512994e3b098d91227 Mon Sep 17 00:00:00 2001 From: machine424 Date: Mon, 8 Jul 2024 18:53:17 +0200 Subject: [PATCH] chore(storage/remote): collect maxTimestamp when value is 0 as well. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change enables the PrometheusRemoteWriteBehind alert’s expression to be evaluated even when the remote endpoint has never been reached. As a result, PrometheusRemoteWriteBehind will fire to easily detect configuration mistakes (such as incorrect endpoint URLs) or unrecoverable connectivity issues. See https://github.com/prometheus/prometheus/issues/14350 for details. Signed-off-by: machine424 --- storage/remote/max_timestamp.go | 6 ------ storage/remote/queue_manager.go | 2 +- storage/remote/queue_manager_test.go | 2 +- storage/remote/write.go | 2 +- 4 files changed, 3 insertions(+), 9 deletions(-) diff --git a/storage/remote/max_timestamp.go b/storage/remote/max_timestamp.go index 3a0a6d6fd4..bb67d9bb98 100644 --- a/storage/remote/max_timestamp.go +++ b/storage/remote/max_timestamp.go @@ -39,9 +39,3 @@ func (m *maxTimestamp) Get() float64 { defer m.mtx.Unlock() return m.value } - -func (m *maxTimestamp) Collect(c chan<- prometheus.Metric) { - if m.Get() > 0 { - m.Gauge.Collect(c) - } -} diff --git a/storage/remote/queue_manager.go b/storage/remote/queue_manager.go index fb13da70da..4bef9909c7 100644 --- a/storage/remote/queue_manager.go +++ b/storage/remote/queue_manager.go @@ -232,7 +232,7 @@ func newQueueManagerMetrics(r prometheus.Registerer, rn, e string) *queueManager Namespace: namespace, Subsystem: subsystem, Name: "queue_highest_sent_timestamp_seconds", - Help: "Timestamp from a WAL sample, the highest timestamp successfully sent by this queue, in seconds since epoch.", + Help: "Timestamp from a WAL sample, the highest timestamp successfully sent by this queue, in seconds since epoch. Initialized to 0 when no data has been sent yet.", ConstLabels: constLabels, }), } diff --git a/storage/remote/queue_manager_test.go b/storage/remote/queue_manager_test.go index 9ab563edab..5227c2d6a7 100644 --- a/storage/remote/queue_manager_test.go +++ b/storage/remote/queue_manager_test.go @@ -60,7 +60,7 @@ func newHighestTimestampMetric() *maxTimestamp { Namespace: namespace, Subsystem: subsystem, Name: "highest_timestamp_in_seconds", - Help: "Highest timestamp that has come into the remote storage via the Appender interface, in seconds since epoch.", + Help: "Highest timestamp that has come into the remote storage via the Appender interface, in seconds since epoch. Initialized to 0 when no data has been received yet", }), } } diff --git a/storage/remote/write.go b/storage/remote/write.go index cd8cd588ca..81902a8f1a 100644 --- a/storage/remote/write.go +++ b/storage/remote/write.go @@ -100,7 +100,7 @@ func NewWriteStorage(logger log.Logger, reg prometheus.Registerer, dir string, f Namespace: namespace, Subsystem: subsystem, Name: "highest_timestamp_in_seconds", - Help: "Highest timestamp that has come into the remote storage via the Appender interface, in seconds since epoch.", + Help: "Highest timestamp that has come into the remote storage via the Appender interface, in seconds since epoch. Initialized to 0 when no data has been received yet.", }), }, }