From 41630b8e8835585098f345b0d9740d7107ffb6eb Mon Sep 17 00:00:00 2001 From: Alban Hurtaud Date: Fri, 6 May 2022 00:42:04 +0200 Subject: [PATCH] Add hidden flag to configure discovery loop interval (#10634) * Add hidden flag to configure discovery loop interval Signed-off-by: Alban HURTAUD --- cmd/prometheus/main.go | 3 +++ scrape/manager.go | 11 ++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index c0848c766a..4c6d2c4a89 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -387,6 +387,9 @@ func main() { serverOnlyFlag(a, "query.max-samples", "Maximum number of samples a single query can load into memory. Note that queries will fail if they try to load more samples than this into memory, so this also limits the number of samples a query can return."). Default("50000000").IntVar(&cfg.queryMaxSamples) + a.Flag("scrape.discovery-reload-interval", "Interval used by scrape manager to throttle target groups updates."). + Hidden().Default("5s").SetValue(&cfg.scrape.DiscoveryReloadInterval) + a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: agent, exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-at-modifier, promql-negative-offset, promql-per-step-stats, remote-write-receiver (DEPRECATED), extra-scrape-metrics, new-service-discovery-manager, auto-gomaxprocs. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details."). Default("").StringsVar(&cfg.featureList) diff --git a/scrape/manager.go b/scrape/manager.go index dc85f17bb5..92ecbc31d4 100644 --- a/scrape/manager.go +++ b/scrape/manager.go @@ -25,6 +25,7 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" config_util "github.com/prometheus/common/config" + "github.com/prometheus/common/model" "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/discovery/targetgroup" @@ -127,6 +128,8 @@ type Options struct { // Option used by downstream scraper users like OpenTelemetry Collector // to help lookup metric metadata. Should be false for Prometheus. PassMetadataInContext bool + // Option to increase the interval used by scrape manager to throttle target groups updates. + DiscoveryReloadInterval model.Duration // Optional HTTP client options to use when scraping. HTTPClientOptions []config_util.HTTPClientOption @@ -170,7 +173,13 @@ func (m *Manager) Run(tsets <-chan map[string][]*targetgroup.Group) error { } func (m *Manager) reloader() { - ticker := time.NewTicker(5 * time.Second) + reloadIntervalDuration := m.opts.DiscoveryReloadInterval + if reloadIntervalDuration < model.Duration(5*time.Second) { + reloadIntervalDuration = model.Duration(5 * time.Second) + } + + ticker := time.NewTicker(time.Duration(reloadIntervalDuration)) + defer ticker.Stop() for {