From b6aaea22fbbab8e68271a6c1fe01fe3ae57dea18 Mon Sep 17 00:00:00 2001 From: Graham Reed Date: Thu, 1 May 2025 08:58:12 +0100 Subject: [PATCH] promtool: Optional fuzzy float64 comparison in rules unittests (#16395) Make fuzzy compare opt-in via fuzzy_compare boolean in each unittest file. Signed-off-by: Graham Reed --- cmd/promtool/testdata/rules_run_fuzzy.yml | 43 ++++++++++++++++++++ cmd/promtool/testdata/rules_run_no_fuzzy.yml | 24 +++++++++++ cmd/promtool/unittest.go | 18 ++++++-- cmd/promtool/unittest_test.go | 23 +++++++++++ docs/configuration/unit_testing_rules.md | 16 +++++--- 5 files changed, 114 insertions(+), 10 deletions(-) create mode 100644 cmd/promtool/testdata/rules_run_fuzzy.yml create mode 100644 cmd/promtool/testdata/rules_run_no_fuzzy.yml diff --git a/cmd/promtool/testdata/rules_run_fuzzy.yml b/cmd/promtool/testdata/rules_run_fuzzy.yml new file mode 100644 index 0000000000..3bf4e47a45 --- /dev/null +++ b/cmd/promtool/testdata/rules_run_fuzzy.yml @@ -0,0 +1,43 @@ +# Minimal test case to see that fuzzy compare is working as expected. +# It should allow slight floating point differences through. Larger +# floating point differences should still fail. + +evaluation_interval: 1m +fuzzy_compare: true + +tests: + - name: correct fuzzy match + input_series: + - series: test_low + values: 2.9999999999999996 + - series: test_high + values: 3.0000000000000004 + promql_expr_test: + - expr: test_low + eval_time: 0 + exp_samples: + - labels: test_low + value: 3 + - expr: test_high + eval_time: 0 + exp_samples: + - labels: test_high + value: 3 + + - name: wrong fuzzy match + input_series: + - series: test_low + values: 2.9999999999999987 + - series: test_high + values: 3.0000000000000013 + promql_expr_test: + - expr: test_low + eval_time: 0 + exp_samples: + - labels: test_low + value: 3 + - expr: test_high + eval_time: 0 + exp_samples: + - labels: test_high + value: 3 diff --git a/cmd/promtool/testdata/rules_run_no_fuzzy.yml b/cmd/promtool/testdata/rules_run_no_fuzzy.yml new file mode 100644 index 0000000000..eba201a28c --- /dev/null +++ b/cmd/promtool/testdata/rules_run_no_fuzzy.yml @@ -0,0 +1,24 @@ +# Minimal test case to see that fuzzy compare can be turned off, +# and slight floating point differences fail matching. + +evaluation_interval: 1m +fuzzy_compare: false + +tests: + - name: correct fuzzy match + input_series: + - series: test_low + values: 2.9999999999999996 + - series: test_high + values: 3.0000000000000004 + promql_expr_test: + - expr: test_low + eval_time: 0 + exp_samples: + - labels: test_low + value: 3 + - expr: test_high + eval_time: 0 + exp_samples: + - labels: test_high + value: 3 diff --git a/cmd/promtool/unittest.go b/cmd/promtool/unittest.go index 7a97a466a6..9bc1af1f61 100644 --- a/cmd/promtool/unittest.go +++ b/cmd/promtool/unittest.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "io" + "math" "os" "path/filepath" "sort" @@ -130,7 +131,7 @@ func ruleUnitTest(filename string, queryOpts promqltest.LazyLoaderOpts, run *reg if t.Interval == 0 { t.Interval = unitTestInp.EvaluationInterval } - ers := t.test(testname, evalInterval, groupOrderMap, queryOpts, diffFlag, debug, ignoreUnknownFields, unitTestInp.RuleFiles...) + ers := t.test(testname, evalInterval, groupOrderMap, queryOpts, diffFlag, debug, ignoreUnknownFields, unitTestInp.FuzzyCompare, unitTestInp.RuleFiles...) if ers != nil { for _, e := range ers { tc.Fail(e.Error()) @@ -159,6 +160,7 @@ type unitTestFile struct { EvaluationInterval model.Duration `yaml:"evaluation_interval,omitempty"` GroupEvalOrder []string `yaml:"group_eval_order"` Tests []testGroup `yaml:"tests"` + FuzzyCompare bool `yaml:"fuzzy_compare,omitempty"` } // resolveAndGlobFilepaths joins all relative paths in a configuration @@ -197,7 +199,7 @@ type testGroup struct { } // test performs the unit tests. -func (tg *testGroup) test(testname string, evalInterval time.Duration, groupOrderMap map[string]int, queryOpts promqltest.LazyLoaderOpts, diffFlag, debug, ignoreUnknownFields bool, ruleFiles ...string) (outErr []error) { +func (tg *testGroup) test(testname string, evalInterval time.Duration, groupOrderMap map[string]int, queryOpts promqltest.LazyLoaderOpts, diffFlag, debug, ignoreUnknownFields, fuzzyCompare bool, ruleFiles ...string) (outErr []error) { if debug { testStart := time.Now() fmt.Printf("DEBUG: Starting test %s\n", testname) @@ -237,6 +239,14 @@ func (tg *testGroup) test(testname string, evalInterval time.Duration, groupOrde mint := time.Unix(0, 0).UTC() maxt := mint.Add(tg.maxEvalTime()) + // Optional floating point compare fuzzing. + var compareFloat64 cmp.Option = cmp.Options{} + if fuzzyCompare { + compareFloat64 = cmp.Comparer(func(x, y float64) bool { + return x == y || math.Nextafter(x, math.Inf(-1)) == y || math.Nextafter(x, math.Inf(1)) == y + }) + } + // Pre-processing some data for testing alerts. // All this preparation is so that we can test alerts as we evaluate the rules. // This avoids storing them in memory, as the number of evals might be high. @@ -374,7 +384,7 @@ func (tg *testGroup) test(testname string, evalInterval time.Duration, groupOrde sort.Sort(gotAlerts) sort.Sort(expAlerts) - if !cmp.Equal(expAlerts, gotAlerts, cmp.Comparer(labels.Equal)) { + if !cmp.Equal(expAlerts, gotAlerts, cmp.Comparer(labels.Equal), compareFloat64) { var testName string if tg.TestGroupName != "" { testName = fmt.Sprintf(" name: %s,\n", tg.TestGroupName) @@ -482,7 +492,7 @@ Outer: sort.Slice(gotSamples, func(i, j int) bool { return labels.Compare(gotSamples[i].Labels, gotSamples[j].Labels) <= 0 }) - if !cmp.Equal(expSamples, gotSamples, cmp.Comparer(labels.Equal)) { + if !cmp.Equal(expSamples, gotSamples, cmp.Comparer(labels.Equal), compareFloat64) { errs = append(errs, fmt.Errorf(" expr: %q, time: %s,\n exp: %v\n got: %v", testCase.Expr, testCase.EvalTime.String(), parsedSamplesString(expSamples), parsedSamplesString(gotSamples))) } diff --git a/cmd/promtool/unittest_test.go b/cmd/promtool/unittest_test.go index 7466b222ca..566e0acbc6 100644 --- a/cmd/promtool/unittest_test.go +++ b/cmd/promtool/unittest_test.go @@ -240,6 +240,29 @@ func TestRulesUnitTestRun(t *testing.T) { ignoreUnknownFields: true, want: 0, }, + { + name: "Test precise floating point comparison expected failure", + args: args{ + files: []string{"./testdata/rules_run_no_fuzzy.yml"}, + }, + want: 1, + }, + { + name: "Test fuzzy floating point comparison correct match", + args: args{ + run: []string{"correct"}, + files: []string{"./testdata/rules_run_fuzzy.yml"}, + }, + want: 0, + }, + { + name: "Test fuzzy floating point comparison wrong match", + args: args{ + run: []string{"wrong"}, + files: []string{"./testdata/rules_run_fuzzy.yml"}, + }, + want: 1, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/docs/configuration/unit_testing_rules.md b/docs/configuration/unit_testing_rules.md index 7fc676a251..ccf1961f48 100644 --- a/docs/configuration/unit_testing_rules.md +++ b/docs/configuration/unit_testing_rules.md @@ -24,6 +24,10 @@ rule_files: [ evaluation_interval: | default = 1m ] +# Setting fuzzy_compare true will very slightly weaken floating point comparisons. +# This will (effectively) ignore differences in the last bit of the mantissa. +[ fuzzy_compare: | default = false ] + # The order in which group names are listed below will be the order of evaluation of # rule groups (at a given evaluation time). The order is guaranteed only for the groups mentioned below. # All the groups need not be mentioned below. @@ -95,20 +99,20 @@ series: # {{schema:1 sum:-0.3 count:3.1 z_bucket:7.1 z_bucket_w:0.05 buckets:[5.1 10 7] offset:-3 n_buckets:[4.1 5] n_offset:-5 counter_reset_hint:gauge}} # Native histograms support the same expanding notation as floating point numbers, i.e. 'axn', 'a+bxn' and 'a-bxn'. # All properties are optional and default to 0. The order is not important. The following properties are supported: -# - schema (int): +# - schema (int): # Currently valid schema numbers are -4 <= n <= 8. They are all for # base-2 bucket schemas, where 1 is a bucket boundary in each case, and # then each power of two is divided into 2^n logarithmic buckets. Or # in other words, each bucket boundary is the previous boundary times # 2^(2^-n). -# - sum (float): +# - sum (float): # The sum of all observations, including the zero bucket. -# - count (non-negative float): +# - count (non-negative float): # The number of observations, including those that are NaN and including the zero bucket. -# - z_bucket (non-negative float): +# - z_bucket (non-negative float): # The sum of all observations in the zero bucket. -# - z_bucket_w (non-negative float): -# The width of the zero bucket. +# - z_bucket_w (non-negative float): +# The width of the zero bucket. # If z_bucket_w > 0, the zero bucket contains all observations -z_bucket_w <= x <= z_bucket_w. # Otherwise, the zero bucket only contains observations that are exactly 0. # - buckets (list of non-negative floats):