Merge pull request #12206 from machadovilaca/add-focus-flag-to-promtool-test-rules

Add '--run' flag to promtool test rules

cmd/promtool/main.go

@@ -198,6 +198,7 @@ func main() {
 
 	testCmd := app.Command("test", "Unit testing.")
 	testRulesCmd := testCmd.Command("rules", "Unit tests for rules.")
+	testRulesRun := testRulesCmd.Flag("run", "If set, will only run test groups whose names match the regular expression. Can be specified multiple times.").Strings()
 	testRulesFiles := testRulesCmd.Arg(
 		"test-rule-file",
 		"The unit test file.",
@@ -366,6 +367,7 @@ func main() {
 				EnableAtModifier:     true,
 				EnableNegativeOffset: true,
 			},
+			*testRulesRun,
 			*testRulesFiles...),
 		)
 

cmd/promtool/testdata/rules_run.yml

@@ -0,0 +1,30 @@
+rule_files:
+  - rules.yml
+
+evaluation_interval: 1m
+
+# Minimal test cases to check focus on a rule group.
+tests:
+  - name: correct test
+    input_series:
+      - series: test
+        values: 1
+
+    promql_expr_test:
+      - expr: test
+        eval_time: 0
+        exp_samples:
+          - value: 1
+            labels: test
+
+  - name: wrong test
+    input_series:
+      - series: test
+        values: 0
+
+    promql_expr_test:
+      - expr: test
+        eval_time: 0
+        exp_samples:
+          - value: 1
+            labels: test

cmd/promtool/unittest.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/grafana/regexp"
 	"github.com/prometheus/common/model"
 	"gopkg.in/yaml.v2"
 
@@ -39,11 +40,16 @@ import (
 
 // RulesUnitTest does unit testing of rules based on the unit testing files provided.
 // More info about the file format can be found in the docs.
-func RulesUnitTest(queryOpts promql.LazyLoaderOpts, files ...string) int {
+func RulesUnitTest(queryOpts promql.LazyLoaderOpts, runStrings []string, files ...string) int {
 	failed := false
 
+	var run *regexp.Regexp
+	if runStrings != nil {
+		run = regexp.MustCompile(strings.Join(runStrings, "|"))
+	}
+
 	for _, f := range files {
-		if errs := ruleUnitTest(f, queryOpts); errs != nil {
+		if errs := ruleUnitTest(f, queryOpts, run); errs != nil {
 			fmt.Fprintln(os.Stderr, "  FAILED:")
 			for _, e := range errs {
 				fmt.Fprintln(os.Stderr, e.Error())
@@ -61,7 +67,7 @@ func RulesUnitTest(queryOpts promql.LazyLoaderOpts, files ...string) int {
 	return successExitCode
 }
 
-func ruleUnitTest(filename string, queryOpts promql.LazyLoaderOpts) []error {
+func ruleUnitTest(filename string, queryOpts promql.LazyLoaderOpts, run *regexp.Regexp) []error {
 	fmt.Println("Unit Testing: ", filename)
 
 	b, err := os.ReadFile(filename)
@@ -96,6 +102,10 @@ func ruleUnitTest(filename string, queryOpts promql.LazyLoaderOpts) []error {
 	// Testing.
 	var errs []error
 	for _, t := range unitTestInp.Tests {
+		if !matchesRun(t.TestGroupName, run) {
+			continue
+		}
+
 		if t.Interval == 0 {
 			t.Interval = unitTestInp.EvaluationInterval
 		}
@@ -111,6 +121,14 @@ func ruleUnitTest(filename string, queryOpts promql.LazyLoaderOpts) []error {
 	return nil
 }
 
+func matchesRun(name string, run *regexp.Regexp) bool {
+	if run == nil {
+		return true
+	}
+
+	return run.MatchString(name)
+}
+
 // unitTestFile holds the contents of a single unit test file.
 type unitTestFile struct {
 	RuleFiles          []string       `yaml:"rule_files"`

cmd/promtool/unittest_test.go

@@ -125,7 +125,60 @@ func TestRulesUnitTest(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := RulesUnitTest(tt.queryOpts, tt.args.files...); got != tt.want {
+			if got := RulesUnitTest(tt.queryOpts, nil, tt.args.files...); got != tt.want {
+				t.Errorf("RulesUnitTest() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestRulesUnitTestRun(t *testing.T) {
+	type args struct {
+		run   []string
+		files []string
+	}
+	tests := []struct {
+		name      string
+		args      args
+		queryOpts promql.LazyLoaderOpts
+		want      int
+	}{
+		{
+			name: "Test all without run arg",
+			args: args{
+				run:   nil,
+				files: []string{"./testdata/rules_run.yml"},
+			},
+			want: 1,
+		},
+		{
+			name: "Test all with run arg",
+			args: args{
+				run:   []string{"correct", "wrong"},
+				files: []string{"./testdata/rules_run.yml"},
+			},
+			want: 1,
+		},
+		{
+			name: "Test correct",
+			args: args{
+				run:   []string{"correct"},
+				files: []string{"./testdata/rules_run.yml"},
+			},
+			want: 0,
+		},
+		{
+			name: "Test wrong",
+			args: args{
+				run:   []string{"wrong"},
+				files: []string{"./testdata/rules_run.yml"},
+			},
+			want: 1,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := RulesUnitTest(tt.queryOpts, tt.args.run, tt.args.files...); got != tt.want {
 				t.Errorf("RulesUnitTest() = %v, want %v", got, tt.want)
 			}
 		})

docs/command-line/promtool.md

@@ -429,6 +429,15 @@ Unit tests for rules.
 
 
 
+###### Flags
+
+| Flag | Description |
+| --- | --- |
+| <code class="text-nowrap">--run</code> | If set, will only run test groups whose names match the regular expression. Can be specified multiple times. |
+
+
+
+
 ###### Arguments
 
 | Argument | Description | Required |