revert unexpected metadata metric fopr RWV2 and add log on unexpected metadata instead. (#17082)

Signed-off-by: leegin <leegin.t@gmail.com>
This commit is contained in:
Darkknight 2025-08-27 00:24:14 +05:30 committed by GitHub
parent 20580b6ba8
commit 9fc4212214
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

View File

@ -64,36 +64,35 @@ const (
type queueManagerMetrics struct {
reg prometheus.Registerer
samplesTotal prometheus.Counter
exemplarsTotal prometheus.Counter
histogramsTotal prometheus.Counter
metadataTotal prometheus.Counter
failedSamplesTotal prometheus.Counter
failedExemplarsTotal prometheus.Counter
failedHistogramsTotal prometheus.Counter
failedMetadataTotal prometheus.Counter
retriedSamplesTotal prometheus.Counter
retriedExemplarsTotal prometheus.Counter
retriedHistogramsTotal prometheus.Counter
retriedMetadataTotal prometheus.Counter
droppedSamplesTotal *prometheus.CounterVec
droppedExemplarsTotal *prometheus.CounterVec
droppedHistogramsTotal *prometheus.CounterVec
enqueueRetriesTotal prometheus.Counter
sentBatchDuration prometheus.Histogram
highestSentTimestamp *maxTimestamp
pendingSamples prometheus.Gauge
pendingExemplars prometheus.Gauge
pendingHistograms prometheus.Gauge
shardCapacity prometheus.Gauge
numShards prometheus.Gauge
maxNumShards prometheus.Gauge
minNumShards prometheus.Gauge
desiredNumShards prometheus.Gauge
sentBytesTotal prometheus.Counter
metadataBytesTotal prometheus.Counter
maxSamplesPerSend prometheus.Gauge
unexpectedMetadataTotal prometheus.Counter
samplesTotal prometheus.Counter
exemplarsTotal prometheus.Counter
histogramsTotal prometheus.Counter
metadataTotal prometheus.Counter
failedSamplesTotal prometheus.Counter
failedExemplarsTotal prometheus.Counter
failedHistogramsTotal prometheus.Counter
failedMetadataTotal prometheus.Counter
retriedSamplesTotal prometheus.Counter
retriedExemplarsTotal prometheus.Counter
retriedHistogramsTotal prometheus.Counter
retriedMetadataTotal prometheus.Counter
droppedSamplesTotal *prometheus.CounterVec
droppedExemplarsTotal *prometheus.CounterVec
droppedHistogramsTotal *prometheus.CounterVec
enqueueRetriesTotal prometheus.Counter
sentBatchDuration prometheus.Histogram
highestSentTimestamp *maxTimestamp
pendingSamples prometheus.Gauge
pendingExemplars prometheus.Gauge
pendingHistograms prometheus.Gauge
shardCapacity prometheus.Gauge
numShards prometheus.Gauge
maxNumShards prometheus.Gauge
minNumShards prometheus.Gauge
desiredNumShards prometheus.Gauge
sentBytesTotal prometheus.Counter
metadataBytesTotal prometheus.Counter
maxSamplesPerSend prometheus.Gauge
}
func newQueueManagerMetrics(r prometheus.Registerer, rn, e string) *queueManagerMetrics {
@ -314,13 +313,6 @@ func newQueueManagerMetrics(r prometheus.Registerer, rn, e string) *queueManager
Help: "The maximum number of samples to be sent, in a single request, to the remote storage. Note that, when sending of exemplars over remote write is enabled, exemplars count towards this limt.",
ConstLabels: constLabels,
})
m.unexpectedMetadataTotal = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "unexpected_metadata_total",
Help: "Total number of unexpected metadata entries in populateV2TimeSeries indicating routing bugs.",
ConstLabels: constLabels,
})
return m
}
@ -357,7 +349,6 @@ func (m *queueManagerMetrics) register() {
m.sentBytesTotal,
m.metadataBytesTotal,
m.maxSamplesPerSend,
m.unexpectedMetadataTotal,
)
}
}
@ -393,7 +384,6 @@ func (m *queueManagerMetrics) unregister() {
m.reg.Unregister(m.sentBytesTotal)
m.reg.Unregister(m.metadataBytesTotal)
m.reg.Unregister(m.maxSamplesPerSend)
m.reg.Unregister(m.unexpectedMetadataTotal)
}
}
@ -1556,7 +1546,7 @@ func (s *shards) runShard(ctx context.Context, shardID int, queue *queue) {
nPendingSamples, nPendingExemplars, nPendingHistograms, nPendingMetadata, nUnexpectedMetadata := populateV2TimeSeries(&symbolTable, batch, pendingDataV2, s.qm.sendExemplars, s.qm.sendNativeHistograms)
n := nPendingSamples + nPendingExemplars + nPendingHistograms
if nUnexpectedMetadata > 0 {
s.qm.metrics.unexpectedMetadataTotal.Add(float64(nUnexpectedMetadata))
s.qm.logger.Warn("unexpected metadata sType in populateV2TimeSeries", "count", nUnexpectedMetadata)
}
_ = s.sendV2Samples(ctx, pendingDataV2[:n], symbolTable.Symbols(), nPendingSamples, nPendingExemplars, nPendingHistograms, nPendingMetadata, &pBufRaw, encBuf, compr)
symbolTable.Reset()