Add global config option for convert_classic_histograms_to_nhcb

Addresses the global part of https://github.com/prometheus/prometheus/issues/13532

Signed-off-by: chardch <otwordsne@gmail.com>

config/config.go

@@ -167,7 +167,8 @@ var (
 		RuleQueryOffset:    model.Duration(0 * time.Minute),
 		// When native histogram feature flag is enabled, ScrapeProtocols default
 		// changes to DefaultNativeHistogramScrapeProtocols.
-		ScrapeProtocols: DefaultScrapeProtocols,
+		ScrapeProtocols:                DefaultScrapeProtocols,
+		ConvertClassicHistogramsToNHCB: false,
 	}
 
 	DefaultRuntimeConfig = RuntimeConfig{
@@ -486,6 +487,8 @@ type GlobalConfig struct {
 	// blank in config files but must have a value if a ScrepeConfig is created
 	// programmatically.
 	MetricNameEscapingScheme string `yaml:"metric_name_escaping_scheme,omitempty"`
+	// Whether to convert all scraped classic histograms into native histograms with custom buckets.
+	ConvertClassicHistogramsToNHCB bool `yaml:"convert_classic_histograms_to_nhcb,omitempty"`
 }
 
 // ScrapeProtocol represents supported protocol for scraping metrics.
@@ -641,7 +644,8 @@ func (c *GlobalConfig) isZero() bool {
 		c.RuleQueryOffset == 0 &&
 		c.QueryLogFile == "" &&
 		c.ScrapeFailureLogFile == "" &&
-		c.ScrapeProtocols == nil
+		c.ScrapeProtocols == nil &&
+		!c.ConvertClassicHistogramsToNHCB
 }
 
 // RuntimeConfig configures the values for the process behavior.
@@ -688,7 +692,7 @@ type ScrapeConfig struct {
 	// Whether to scrape a classic histogram, even if it is also exposed as a native histogram.
 	AlwaysScrapeClassicHistograms bool `yaml:"always_scrape_classic_histograms,omitempty"`
 	// Whether to convert all scraped classic histograms into a native histogram with custom buckets.
-	ConvertClassicHistogramsToNHCB bool `yaml:"convert_classic_histograms_to_nhcb,omitempty"`
+	ConvertClassicHistogramsToNHCB *bool `yaml:"convert_classic_histograms_to_nhcb,omitempty"`
 	// File to which scrape failures are logged.
 	ScrapeFailureLogFile string `yaml:"scrape_failure_log_file,omitempty"`
 	// The HTTP resource path on which to fetch metrics from targets.
@@ -895,6 +899,11 @@ func (c *ScrapeConfig) Validate(globalConfig GlobalConfig) error {
 		return fmt.Errorf("unknown scrape config name escaping method specified, must be one of '%s', '%s', '%s', or '%s', got %s", model.AllowUTF8, model.EscapeUnderscores, model.EscapeDots, model.EscapeValues, c.MetricNameValidationScheme)
 	}
 
+	if c.ConvertClassicHistogramsToNHCB == nil {
+		globalVal := &globalConfig.ConvertClassicHistogramsToNHCB
+		c.ConvertClassicHistogramsToNHCB = globalVal
+	}
+
 	return nil
 }
 
@@ -917,6 +926,11 @@ func ToValidationScheme(s string) (validationScheme model.ValidationScheme, err
 	return validationScheme, nil
 }
 
+// ConvertClassicHistogramsToNHCBEnabled returns whether to convert classic histograms to NHCB
+func (c *ScrapeConfig) ConvertClassicHistogramsToNHCBEnabled() bool {
+	return c.ConvertClassicHistogramsToNHCB != nil && *c.ConvertClassicHistogramsToNHCB
+}
+
 // StorageConfig configures runtime reloadable configuration options.
 type StorageConfig struct {
 	TSDBConfig      *TSDBConfig      `yaml:"tsdb,omitempty"`

config/testdata/global_convert_classic_hist_to_nhcb.good.yml

@@ -0,0 +1,6 @@
+global:
+  convert_classic_histograms_to_nhcb: true
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: ['localhost:8080']

config/testdata/local_convert_classic_hist_to_nhcb.good.yml

@@ -0,0 +1,7 @@
+global:
+  convert_classic_histograms_to_nhcb: false
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: ['localhost:8080']
+    convert_classic_histograms_to_nhcb: true

config/testdata/local_disable_convert_classic_hist_to_nhcb.good.yml

@@ -0,0 +1,7 @@
+global:
+  convert_classic_histograms_to_nhcb: true
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: ['localhost:8080']
+    convert_classic_histograms_to_nhcb: false

docs/configuration/configuration.md

@@ -140,6 +140,10 @@ global:
   # and underscores.
   [ metric_name_validation_scheme <string> | default "utf8" ]
 
+  # Specifies whether to convert all scraped classic histograms into native
+  # histograms with custom buckets.
+  [ convert_classic_histograms_to_nhcb <bool> | default = false]
+
 runtime:
   # Configure the Go garbage collector GOGC parameter
   # See: https://tip.golang.org/doc/gc-guide#GOGC
@@ -531,6 +535,11 @@ metric_relabel_configs:
 # 0 results in the smallest supported factor (which is currently ~1.0027 or
 # schema 8, but might change in the future).
 [ native_histogram_min_bucket_factor: <float> | default = 0 ]
+
+# Specifies whether to convert scraped classic histograms from this into
+# native histogram with custom buckets.
+[ convert_classic_histograms_to_nhcb <bool> | default = false 
+or global.convert_classic_histograms_to_nhcb if unset]
 ```
 
 Where `<job_name>` must be unique across all scrape configurations.

scrape/manager.go

@@ -176,7 +176,7 @@ func (m *Manager) reload() {
 				m.logger.Error("error reloading target set", "err", "invalid config id:"+setName)
 				continue
 			}
-			if scrapeConfig.ConvertClassicHistogramsToNHCB && m.opts.EnableCreatedTimestampZeroIngestion {
+			if scrapeConfig.ConvertClassicHistogramsToNHCBEnabled() && m.opts.EnableCreatedTimestampZeroIngestion {
 				// TODO(krajorama): fix https://github.com/prometheus/prometheus/issues/15137
 				m.logger.Error("error reloading target set", "err", "cannot convert classic histograms to native histograms with custom buckets and ingest created timestamp zero samples at the same time due to https://github.com/prometheus/prometheus/issues/15137")
 				continue

scrape/scrape.go

@@ -367,7 +367,7 @@ func (sp *scrapePool) restartLoops(reuseCache bool) {
 		mrc                      = sp.config.MetricRelabelConfigs
 		fallbackScrapeProtocol   = sp.config.ScrapeFallbackProtocol.HeaderMediaType()
 		alwaysScrapeClassicHist  = sp.config.AlwaysScrapeClassicHistograms
-		convertClassicHistToNHCB = sp.config.ConvertClassicHistogramsToNHCB
+		convertClassicHistToNHCB = sp.config.ConvertClassicHistogramsToNHCBEnabled()
 	)
 
 	sp.targetMtx.Lock()
@@ -523,7 +523,7 @@ func (sp *scrapePool) sync(targets []*Target) {
 		mrc                      = sp.config.MetricRelabelConfigs
 		fallbackScrapeProtocol   = sp.config.ScrapeFallbackProtocol.HeaderMediaType()
 		alwaysScrapeClassicHist  = sp.config.AlwaysScrapeClassicHistograms
-		convertClassicHistToNHCB = sp.config.ConvertClassicHistogramsToNHCB
+		convertClassicHistToNHCB = sp.config.ConvertClassicHistogramsToNHCB != nil && *sp.config.ConvertClassicHistogramsToNHCB
 	)
 
 	sp.targetMtx.Lock()

scrape/scrape_test.go

@@ -4631,26 +4631,31 @@ metric: <
 		require.Equal(t, expectedCount, count, "number of histogram series not as expected")
 	}
 
+	tru := true
+	fals := false
 	for metricsTextName, metricsText := range metricsTexts {
 		for name, tc := range map[string]struct {
 			alwaysScrapeClassicHistograms bool
-			convertClassicHistToNHCB      bool
+			convertClassicHistToNHCB      *bool
 		}{
 			"convert with scrape": {
 				alwaysScrapeClassicHistograms: true,
-				convertClassicHistToNHCB:      true,
+				convertClassicHistToNHCB:      &tru,
 			},
 			"convert without scrape": {
 				alwaysScrapeClassicHistograms: false,
-				convertClassicHistToNHCB:      true,
+				convertClassicHistToNHCB:      &tru,
 			},
 			"scrape without convert": {
 				alwaysScrapeClassicHistograms: true,
-				convertClassicHistToNHCB:      false,
+				convertClassicHistToNHCB:      &fals,
+			},
+			"scrape with nil convert": {
+				alwaysScrapeClassicHistograms: true,
 			},
 			"neither scrape nor convert": {
 				alwaysScrapeClassicHistograms: false,
-				convertClassicHistToNHCB:      false,
+				convertClassicHistToNHCB:      &fals,
 			},
 		} {
 			var expectedClassicHistCount, expectedNativeHistCount int
@@ -4664,17 +4669,17 @@ metric: <
 				}
 			} else if metricsText.hasClassic {
 				switch {
-				case tc.alwaysScrapeClassicHistograms && tc.convertClassicHistToNHCB:
+				case tc.convertClassicHistToNHCB == nil || !*tc.convertClassicHistToNHCB:
+					expectedClassicHistCount = 1
+					expectedNativeHistCount = 0
+				case tc.alwaysScrapeClassicHistograms && *tc.convertClassicHistToNHCB:
 					expectedClassicHistCount = 1
 					expectedNativeHistCount = 1
 					expectCustomBuckets = true
-				case !tc.alwaysScrapeClassicHistograms && tc.convertClassicHistToNHCB:
+				case !tc.alwaysScrapeClassicHistograms && *tc.convertClassicHistToNHCB:
 					expectedClassicHistCount = 0
 					expectedNativeHistCount = 1
 					expectCustomBuckets = true
-				case !tc.convertClassicHistToNHCB:
-					expectedClassicHistCount = 1
-					expectedNativeHistCount = 0
 				}
 			}