Add script for converting PromQL tests to new syntax format (#16562)

Signed-off-by: Kapil Lamba <kapillamba4@gmail.com>
Co-authored-by: Neeraj Gartia <80708727+NeerajGartia21@users.noreply.github.com>

promql/promqltest/README.md

@@ -164,3 +164,30 @@ There can be multiple `<expect>` lines for a given `<type>`. Each `<type>` valid
 Every `<expect>` line must match at least one corresponding annotation or error.
 
 If at least one `<expect>` line of type `warn` or `info` is present, then all corresponding annotations must have a matching `expect` line.
+
+#### Migrating Test Files to the New Syntax
+
+- All `.test` files in the directory specified by the --dir flag will be updated in place.
+- Deprecated syntax will be replaced with the recommended `expect` line statements.
+
+Usage:
+```sh
+go run ./promql/promqltest/cmd/migrate/main.go --mode=strict [--dir=<directory>]
+```
+
+The `--mode` flag controls how expectations are migrated:
+- `strict`: Strictly migrates all expectations to the new syntax.
+  This is probably more verbose than intended because the old syntax
+  implied many constraints that are often not needed.
+- `basic`: Like `strict` but never creates `no_info` and `no_warn`
+  expectations. This can be a good starting point to manually add 
+  `no_info` and `no_warn` expectations and/or remove `info` and 
+  `warn` expectations as needed.
+- `tolerant`: Only creates `expect fail` and `expect ordered` where
+  appropriate. All desired expectations about presence or absence 
+  of `info` and `warn` have to be added manually.
+
+All three modes create valid passing tests from previously passing tests.
+`basic` and `tolerant` just test fewer expectations than the previous tests.
+
+The --dir flag specifies the directory containing test files to migrate.

promql/promqltest/cmd/migrate/main.go

@@ -0,0 +1,33 @@
+// Copyright 2025 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/prometheus/prometheus/promql/promqltest"
+)
+
+func main() {
+	mode := flag.String("mode", "strict", "Migration mode: strict, basic, or tolerant")
+	dir := flag.String("dir", "", "Directory to migrate")
+	flag.Parse()
+
+	if err := promqltest.MigrateTestData(*mode, *dir); err != nil {
+		fmt.Printf("Error migrating test files: %v\n", err)
+		os.Exit(1)
+	}
+}

promql/promqltest/test_migrate.go

@@ -0,0 +1,200 @@
+// Copyright 2025 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package promqltest
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/grafana/regexp"
+)
+
+const defaultTestDataDir = "promql/promqltest/testdata"
+
+var (
+	evalRegex   = regexp.MustCompile(`^(eval |eval_fail |eval_warn |eval_info |eval_ordered )(.*)$`)
+	indentRegex = regexp.MustCompile(`^([ \t]+)\S`)
+)
+
+type MigrateMode int
+
+const (
+	MigrateStrict MigrateMode = iota
+	MigrateBasic
+	MigrateTolerant
+)
+
+func ParseMigrateMode(s string) (MigrateMode, error) {
+	switch s {
+	case "strict":
+		return MigrateStrict, nil
+	case "basic":
+		return MigrateBasic, nil
+	case "tolerant":
+		return MigrateTolerant, nil
+	default:
+		return MigrateStrict, fmt.Errorf("invalid mode: %s", s)
+	}
+}
+
+// MigrateTestData migrates all PromQL test files to the new syntax format.
+// It applies annotation rules based on the provided migration mode ("strict", "basic", or "tolerant").
+// The function parses each .test file, converts it to the new syntax and overwrites the file.
+func MigrateTestData(mode, dir string) error {
+	if dir == "" {
+		dir = defaultTestDataDir
+	}
+
+	migrationMode, err := ParseMigrateMode(mode)
+	if err != nil {
+		return fmt.Errorf("failed to parse mode: %w", err)
+	}
+
+	files, err := os.ReadDir(dir)
+	if err != nil {
+		return fmt.Errorf("failed to read testdata directory: %w", err)
+	}
+
+	annotationMap := map[MigrateMode]map[string][]string{
+		MigrateStrict: {
+			"eval_fail":    {"expect fail", "expect no_warn", "expect no_info"},
+			"eval_warn":    {"expect warn", "expect no_info"},
+			"eval_info":    {"expect info", "expect no_warn"},
+			"eval_ordered": {"expect ordered", "expect no_warn", "expect no_info"},
+			"eval":         {"expect no_warn", "expect no_info"},
+		},
+		MigrateBasic: {
+			"eval_fail":    {"expect fail"},
+			"eval_warn":    {"expect warn"},
+			"eval_info":    {"expect info"},
+			"eval_ordered": {"expect ordered"},
+		},
+		MigrateTolerant: {
+			"eval_fail":    {"expect fail"},
+			"eval_ordered": {"expect ordered"},
+		},
+	}
+
+	for _, file := range files {
+		if file.IsDir() || !strings.HasSuffix(file.Name(), ".test") {
+			continue
+		}
+
+		path := filepath.Join(dir, file.Name())
+		content, err := os.ReadFile(path)
+		if err != nil {
+			return fmt.Errorf("failed to read file %s: %w", path, err)
+		}
+
+		lines := strings.Split(string(content), "\n")
+		processedLines, err := processTestFileLines(lines, annotationMap[migrationMode], evalRegex)
+		if err != nil {
+			return fmt.Errorf("error processing file %s: %w", path, err)
+		}
+
+		if err := os.WriteFile(path, []byte(strings.Join(processedLines, "\n")), 0o644); err != nil {
+			return fmt.Errorf("failed to write file %s: %w", path, err)
+		}
+	}
+	return nil
+}
+
+func processTestFileLines(
+	lines []string,
+	annotationMap map[string][]string,
+	evalRegex *regexp.Regexp,
+) (result []string, err error) {
+	for i := 0; i < len(lines); i++ {
+		startLine := lines[i]
+		matches := evalRegex.FindStringSubmatch(strings.TrimSpace(startLine))
+		if matches == nil {
+			result = append(result, startLine)
+			continue
+		}
+
+		var inputBlock []string
+		var outputBlock []string
+		skipBlock := false
+		i++
+		for i < len(lines) {
+			inputBlock = append(inputBlock, lines[i])
+			if strings.HasPrefix(strings.TrimSpace(lines[i]), "expect ") {
+				skipBlock = true
+			}
+			if i+1 < len(lines) && evalRegex.MatchString(strings.TrimSpace(lines[i+1])) {
+				break
+			}
+			i++
+		}
+
+		if skipBlock {
+			result = append(result, startLine)
+			result = append(result, inputBlock...)
+			continue
+		}
+
+		// Get leading whitespace from startLine using indentRegex.
+		leadingWS := ""
+		if indentMatch := indentRegex.FindStringSubmatch(startLine); indentMatch != nil {
+			leadingWS = indentMatch[1]
+		}
+
+		command := strings.TrimSpace(matches[1])
+		expression := matches[2]
+		var annotations []string
+		result = append(result, leadingWS+fmt.Sprintf("eval %s", expression))
+
+		// Detecting indentation style (tab or space) from the first non-empty, indented line.
+		indent := "  "
+		for _, line := range inputBlock {
+			if indentMatch := indentRegex.FindStringSubmatch(line); indentMatch != nil {
+				indent = indentMatch[1]
+				break
+			}
+		}
+
+		for _, annotation := range annotationMap[command] {
+			annotations = append(annotations, indent+annotation)
+		}
+
+		for _, line := range inputBlock {
+			trimmedLine := strings.TrimSpace(line)
+			switch {
+			case strings.HasPrefix(trimmedLine, "expected_fail_message"):
+				msg := strings.TrimPrefix(trimmedLine, "expected_fail_message ")
+				for j, s := range annotations {
+					if strings.Contains(s, "expect fail") {
+						annotations[j] = indent + fmt.Sprintf("expect fail msg:%s", msg)
+					}
+				}
+			case strings.HasPrefix(trimmedLine, "expected_fail_regexp"):
+				regex := strings.TrimPrefix(trimmedLine, "expected_fail_regexp ")
+				for j, s := range annotations {
+					if strings.Contains(s, "expect fail") {
+						annotations[j] = indent + fmt.Sprintf("expect fail regex:%s", regex)
+					}
+				}
+			default:
+				outputBlock = append(outputBlock, line)
+			}
+		}
+
+		result = append(result, annotations...)
+		result = append(result, outputBlock...)
+	}
+
+	return result, nil
+}

promql/promqltest/test_migrate_test.go

@@ -0,0 +1,381 @@
+// Copyright 2025 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package promqltest
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func writeTestFile(t *testing.T, dir, content string) string {
+	t.Helper()
+	testFile := filepath.Join(dir, "testcase.test")
+	require.NoError(t, os.WriteFile(testFile, []byte(content), 0o644))
+	return testFile
+}
+
+func readTestFile(t *testing.T, path string) string {
+	t.Helper()
+	output, err := os.ReadFile(path)
+	require.NoError(t, err)
+	return string(output)
+}
+
+func assertMigration(t *testing.T, mode, input, expected string) {
+	dir := t.TempDir()
+	testFile := writeTestFile(t, dir, input)
+
+	err := MigrateTestData(mode, dir)
+	require.NoError(t, err)
+
+	output := readTestFile(t, testFile)
+	require.Equal(t, expected, output)
+}
+
+func TestMigrateTestData_BasicMode(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{
+			name: "Basic mode with fail",
+			input: `
+				eval_fail instant at 1m sum(foo)
+				  expected_fail_message something went wrong
+				  {src="a"} 1
+			`,
+			expected: `
+				eval instant at 1m sum(foo)
+				  expect fail msg:something went wrong
+				  {src="a"} 1
+			`,
+		},
+		{
+			name: "Basic mode with warn",
+			input: `
+				eval_warn instant at 2m avg(bar)
+				  {src="a"} 1
+			`,
+			expected: `
+				eval instant at 2m avg(bar)
+				  expect warn
+				  {src="a"} 1
+			`,
+		},
+		{
+			name: "Basic mode with info",
+			input: `
+				eval_info instant at 3m min(baz)
+				  {src="a"} 1
+			`,
+			expected: `
+				eval instant at 3m min(baz)
+				  expect info
+				  {src="a"} 1
+			`,
+		},
+		{
+			name: "Basic mode with ordered",
+			input: `
+				eval_ordered instant at 4m max(qux)
+				  {src="a"} 1
+			`,
+			expected: `
+				eval instant at 4m max(qux)
+				  expect ordered
+				  {src="a"} 1
+			`,
+		},
+		{
+			name: "Basic mode with multiple eval blocks",
+			input: `
+				eval_fail instant at 1m sum(foo)
+				  expected_fail_message something else went wrong
+				  {src="a"} 1
+
+				eval_warn instant at 2m avg(bar)
+				  {src="a"} 1
+			`,
+			expected: `
+				eval instant at 1m sum(foo)
+				  expect fail msg:something else went wrong
+				  {src="a"} 1
+
+				eval instant at 2m avg(bar)
+				  expect warn
+				  {src="a"} 1
+			`,
+		},
+		{
+			name: "Basic mode with already migrated syntax (no changes)",
+			input: `
+				eval instant at 1m sum(foo)
+				  expect fail msg:something went wrong
+				  {src="a"} 1
+			`,
+			expected: `
+				eval instant at 1m sum(foo)
+				  expect fail msg:something went wrong
+				  {src="a"} 1
+			`,
+		},
+		{
+			name: "Basic mode with only comments and whitespace",
+			input: `
+			# This is a comment
+			
+			`,
+			expected: `
+			# This is a comment
+			
+			`,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			assertMigration(t, "basic", tc.input, tc.expected)
+		})
+	}
+}
+
+func TestMigrateTestData_StrictMode(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{
+			name: "Strict mode with fail",
+			input: `
+				eval_fail instant at 1m sum(foo)
+					expected_fail_message something went wrong
+					{src="a"} 1
+`,
+			expected: `
+				eval instant at 1m sum(foo)
+					expect fail msg:something went wrong
+					expect no_warn
+					expect no_info
+					{src="a"} 1
+`,
+		},
+		{
+			name: "Strict mode with warn",
+			input: `
+				eval_warn instant at 2m avg(bar)
+					{src="a"} 1
+`,
+			expected: `
+				eval instant at 2m avg(bar)
+					expect warn
+					expect no_info
+					{src="a"} 1
+`,
+		},
+		{
+			name: "Strict mode with info",
+			input: `
+				eval_info instant at 3m min(baz)
+					{src="a"} 1
+`,
+			expected: `
+				eval instant at 3m min(baz)
+					expect info
+					expect no_warn
+					{src="a"} 1
+`,
+		},
+		{
+			name: "Strict mode with ordered",
+			input: `
+				eval_ordered instant at 4m max(qux)
+					{src="a"} 1
+`,
+			expected: `
+				eval instant at 4m max(qux)
+					expect ordered
+					expect no_warn
+					expect no_info
+					{src="a"} 1
+`,
+		},
+		{
+			name: "Strict mode with multiple eval blocks",
+			input: `
+				eval_fail instant at 1m sum(foo)
+					expected_fail_message something else went wrong
+					{src="a"} 1
+
+				eval_warn instant at 2m avg(bar)
+					{src="a"} 1
+`,
+			expected: `
+				eval instant at 1m sum(foo)
+					expect fail msg:something else went wrong
+					expect no_warn
+					expect no_info
+					{src="a"} 1
+
+				eval instant at 2m avg(bar)
+					expect warn
+					expect no_info
+					{src="a"} 1
+`,
+		},
+		{
+			name: "Strict mode with already migrated syntax (no changes)",
+			input: `
+				eval instant at 1m sum(foo)
+					expect fail msg:something went wrong
+					expect no_warn
+					expect no_info
+					{src="a"} 1
+`,
+			expected: `
+				eval instant at 1m sum(foo)
+					expect fail msg:something went wrong
+					expect no_warn
+					expect no_info
+					{src="a"} 1
+`,
+		},
+		{
+			name: "Strict mode with only comments and whitespace",
+			input: `
+			# This is a comment
+
+`,
+			expected: `
+			# This is a comment
+
+`,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			assertMigration(t, "strict", tc.input, tc.expected)
+		})
+	}
+}
+
+func TestMigrateTestData_TolerantMode(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{
+			name: "Tolerant mode with fail",
+			input: `
+				eval_fail instant at 1m sum(foo)
+					expected_fail_message something went wrong
+					{src="a"} 1
+`,
+			expected: `
+				eval instant at 1m sum(foo)
+					expect fail msg:something went wrong
+					{src="a"} 1
+`,
+		},
+		{
+			name: "Tolerant mode with warn",
+			input: `
+				eval_warn instant at 2m avg(bar)
+					{src="a"} 1
+`,
+			expected: `
+				eval instant at 2m avg(bar)
+					{src="a"} 1
+`,
+		},
+		{
+			name: "Tolerant mode with info",
+			input: `
+				eval_info instant at 3m min(baz)
+					{src="a"} 1
+`,
+			expected: `
+				eval instant at 3m min(baz)
+					{src="a"} 1
+`,
+		},
+		{
+			name: "Tolerant mode with ordered",
+			input: `
+				eval_ordered instant at 4m max(qux)
+					{src="a"} 1
+`,
+			expected: `
+				eval instant at 4m max(qux)
+					expect ordered
+					{src="a"} 1
+`,
+		},
+		{
+			name: "Tolerant mode with multiple eval blocks",
+			input: `
+				eval_fail instant at 1m sum(foo)
+					expected_fail_message something else went wrong
+					{src="a"} 1
+
+				eval_warn instant at 2m avg(bar)
+					{src="a"} 1
+`,
+			expected: `
+				eval instant at 1m sum(foo)
+					expect fail msg:something else went wrong
+					{src="a"} 1
+
+				eval instant at 2m avg(bar)
+					{src="a"} 1
+`,
+		},
+		{
+			name: "Tolerant mode with already migrated syntax (no changes)",
+			input: `
+				eval instant at 1m sum(foo)
+					expect fail msg:something went wrong
+					{src="a"} 1
+`,
+			expected: `
+				eval instant at 1m sum(foo)
+					expect fail msg:something went wrong
+					{src="a"} 1
+`,
+		},
+		{
+			name: "Tolerant mode with only comments and whitespace",
+			input: `
+			# This is a comment
+
+`,
+			expected: `
+			# This is a comment
+
+`,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			assertMigration(t, "tolerant", tc.input, tc.expected)
+		})
+	}
+}