promtool: Optional fuzzy float64 comparison in rules unittests (#16395)

Make fuzzy compare opt-in via fuzzy_compare boolean in each unittest file.

Signed-off-by: Graham Reed <greed@hypervolt.co.uk>

cmd/promtool/testdata/rules_run_fuzzy.yml

@@ -0,0 +1,43 @@
+# Minimal test case to see that fuzzy compare is working as expected.
+# It should allow slight floating point differences through. Larger
+# floating point differences should still fail.
+
+evaluation_interval: 1m
+fuzzy_compare: true
+
+tests:
+  - name: correct fuzzy match
+    input_series:
+      - series: test_low
+        values: 2.9999999999999996
+      - series: test_high
+        values: 3.0000000000000004
+    promql_expr_test:
+      - expr: test_low
+        eval_time: 0
+        exp_samples:
+          - labels: test_low
+            value: 3
+      - expr: test_high
+        eval_time: 0
+        exp_samples:
+          - labels: test_high
+            value: 3
+
+  - name: wrong fuzzy match
+    input_series:
+      - series: test_low
+        values: 2.9999999999999987
+      - series: test_high
+        values: 3.0000000000000013
+    promql_expr_test:
+      - expr: test_low
+        eval_time: 0
+        exp_samples:
+          - labels: test_low
+            value: 3
+      - expr: test_high
+        eval_time: 0
+        exp_samples:
+          - labels: test_high
+            value: 3

cmd/promtool/testdata/rules_run_no_fuzzy.yml

@@ -0,0 +1,24 @@
+# Minimal test case to see that fuzzy compare can be turned off,
+# and slight floating point differences fail matching.
+
+evaluation_interval: 1m
+fuzzy_compare: false
+
+tests:
+  - name: correct fuzzy match
+    input_series:
+      - series: test_low
+        values: 2.9999999999999996
+      - series: test_high
+        values: 3.0000000000000004
+    promql_expr_test:
+      - expr: test_low
+        eval_time: 0
+        exp_samples:
+          - labels: test_low
+            value: 3
+      - expr: test_high
+        eval_time: 0
+        exp_samples:
+          - labels: test_high
+            value: 3

cmd/promtool/unittest.go

@@ -19,6 +19,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math"
 	"os"
 	"path/filepath"
 	"sort"
@@ -130,7 +131,7 @@ func ruleUnitTest(filename string, queryOpts promqltest.LazyLoaderOpts, run *reg
 		if t.Interval == 0 {
 			t.Interval = unitTestInp.EvaluationInterval
 		}
-		ers := t.test(testname, evalInterval, groupOrderMap, queryOpts, diffFlag, debug, ignoreUnknownFields, unitTestInp.RuleFiles...)
+		ers := t.test(testname, evalInterval, groupOrderMap, queryOpts, diffFlag, debug, ignoreUnknownFields, unitTestInp.FuzzyCompare, unitTestInp.RuleFiles...)
 		if ers != nil {
 			for _, e := range ers {
 				tc.Fail(e.Error())
@@ -159,6 +160,7 @@ type unitTestFile struct {
 	EvaluationInterval model.Duration `yaml:"evaluation_interval,omitempty"`
 	GroupEvalOrder     []string       `yaml:"group_eval_order"`
 	Tests              []testGroup    `yaml:"tests"`
+	FuzzyCompare       bool           `yaml:"fuzzy_compare,omitempty"`
 }
 
 // resolveAndGlobFilepaths joins all relative paths in a configuration
@@ -197,7 +199,7 @@ type testGroup struct {
 }
 
 // test performs the unit tests.
-func (tg *testGroup) test(testname string, evalInterval time.Duration, groupOrderMap map[string]int, queryOpts promqltest.LazyLoaderOpts, diffFlag, debug, ignoreUnknownFields bool, ruleFiles ...string) (outErr []error) {
+func (tg *testGroup) test(testname string, evalInterval time.Duration, groupOrderMap map[string]int, queryOpts promqltest.LazyLoaderOpts, diffFlag, debug, ignoreUnknownFields, fuzzyCompare bool, ruleFiles ...string) (outErr []error) {
 	if debug {
 		testStart := time.Now()
 		fmt.Printf("DEBUG: Starting test %s\n", testname)
@@ -237,6 +239,14 @@ func (tg *testGroup) test(testname string, evalInterval time.Duration, groupOrde
 	mint := time.Unix(0, 0).UTC()
 	maxt := mint.Add(tg.maxEvalTime())
 
+	// Optional floating point compare fuzzing.
+	var compareFloat64 cmp.Option = cmp.Options{}
+	if fuzzyCompare {
+		compareFloat64 = cmp.Comparer(func(x, y float64) bool {
+			return x == y || math.Nextafter(x, math.Inf(-1)) == y || math.Nextafter(x, math.Inf(1)) == y
+		})
+	}
+
 	// Pre-processing some data for testing alerts.
 	// All this preparation is so that we can test alerts as we evaluate the rules.
 	// This avoids storing them in memory, as the number of evals might be high.
@@ -374,7 +384,7 @@ func (tg *testGroup) test(testname string, evalInterval time.Duration, groupOrde
 				sort.Sort(gotAlerts)
 				sort.Sort(expAlerts)
 
-				if !cmp.Equal(expAlerts, gotAlerts, cmp.Comparer(labels.Equal)) {
+				if !cmp.Equal(expAlerts, gotAlerts, cmp.Comparer(labels.Equal), compareFloat64) {
 					var testName string
 					if tg.TestGroupName != "" {
 						testName = fmt.Sprintf("    name: %s,\n", tg.TestGroupName)
@@ -482,7 +492,7 @@ Outer:
 		sort.Slice(gotSamples, func(i, j int) bool {
 			return labels.Compare(gotSamples[i].Labels, gotSamples[j].Labels) <= 0
 		})
-		if !cmp.Equal(expSamples, gotSamples, cmp.Comparer(labels.Equal)) {
+		if !cmp.Equal(expSamples, gotSamples, cmp.Comparer(labels.Equal), compareFloat64) {
 			errs = append(errs, fmt.Errorf("    expr: %q, time: %s,\n        exp: %v\n        got: %v", testCase.Expr,
 				testCase.EvalTime.String(), parsedSamplesString(expSamples), parsedSamplesString(gotSamples)))
 		}

cmd/promtool/unittest_test.go

@@ -240,6 +240,29 @@ func TestRulesUnitTestRun(t *testing.T) {
 			ignoreUnknownFields: true,
 			want:                0,
 		},
+		{
+			name: "Test precise floating point comparison expected failure",
+			args: args{
+				files: []string{"./testdata/rules_run_no_fuzzy.yml"},
+			},
+			want: 1,
+		},
+		{
+			name: "Test fuzzy floating point comparison correct match",
+			args: args{
+				run:   []string{"correct"},
+				files: []string{"./testdata/rules_run_fuzzy.yml"},
+			},
+			want: 0,
+		},
+		{
+			name: "Test fuzzy floating point comparison wrong match",
+			args: args{
+				run:   []string{"wrong"},
+				files: []string{"./testdata/rules_run_fuzzy.yml"},
+			},
+			want: 1,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

docs/configuration/unit_testing_rules.md

@@ -24,6 +24,10 @@ rule_files:
 
 [ evaluation_interval: <duration> | default = 1m ]
 
+# Setting fuzzy_compare true will very slightly weaken floating point comparisons.
+# This will (effectively) ignore differences in the last bit of the mantissa.
+[ fuzzy_compare: <boolean> | default = false ]
+
 # The order in which group names are listed below will be the order of evaluation of
 # rule groups (at a given evaluation time). The order is guaranteed only for the groups mentioned below.
 # All the groups need not be mentioned below.
@@ -95,20 +99,20 @@ series: <string>
 #     {{schema:1 sum:-0.3 count:3.1 z_bucket:7.1 z_bucket_w:0.05 buckets:[5.1 10 7] offset:-3 n_buckets:[4.1 5] n_offset:-5 counter_reset_hint:gauge}}
 #     Native histograms support the same expanding notation as floating point numbers, i.e. 'axn', 'a+bxn' and 'a-bxn'.
 #     All properties are optional and default to 0. The order is not important. The following properties are supported:
-#     - schema (int): 
+#     - schema (int):
 #         Currently valid schema numbers are -4 <= n <= 8. They are all for
 #         base-2 bucket schemas, where 1 is a bucket boundary in each case, and
 #         then each power of two is divided into 2^n logarithmic buckets.  Or
 #         in other words, each bucket boundary is the previous boundary times
 #         2^(2^-n).
-#     - sum (float): 
+#     - sum (float):
 #         The sum of all observations, including the zero bucket.
-#     - count (non-negative float): 
+#     - count (non-negative float):
 #         The number of observations, including those that are NaN and including the zero bucket.
-#     - z_bucket (non-negative float): 
+#     - z_bucket (non-negative float):
 #         The sum of all observations in the zero bucket.
-#     - z_bucket_w (non-negative float): 
+#     - z_bucket_w (non-negative float):
-#         The width of the zero bucket. 
+#         The width of the zero bucket.
 #         If z_bucket_w > 0, the zero bucket contains all observations -z_bucket_w <= x <= z_bucket_w.
 #         Otherwise, the zero bucket only contains observations that are exactly 0.
 #     - buckets (list of non-negative floats):