mirrors/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2025-09-20 21:31:02 +02:00
Code Issues Projects Releases Wiki Activity

Added support for string literals and range results for instant queries in test scripting framework (#17055)

Browse Source 
Signed-off-by: Andrew Hall <andrew.hall@grafana.com>
Co-authored-by: Charles Korn <charleskorn@users.noreply.github.com>
Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>
This commit is contained in:
Andrew Hall 2025-09-16 19:28:19 +08:00 committed by GitHub
parent 26279e5b6d
commit aa922ce3b6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 355 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
promql/engine_test.go
View File

@ -3195,89 +3195,6 @@ func TestEngine_Close(t *testing.T) {
})
}
func TestInstantQueryWithRangeVectorSelector(t *testing.T) {
engine := newTestEngine(t)
baseT := timestamp.Time(0)
storage := promqltest.LoadedStorage(t, `
load 1m
some_metric{env="1"} 0+1x4
some_metric{env="2"} 0+2x4
some_metric{env="3"} {{count:0}}+{{count:1}}x4
some_metric_with_stale_marker 0 1 stale 3
`)
t.Cleanup(func() { require.NoError(t, storage.Close()) })
testCases := map[string]struct {
expr string
expected promql.Matrix
ts time.Time
}{
"matches series with points in range": {
expr: "some_metric[2m]",
ts: baseT.Add(2 * time.Minute),
expected: promql.Matrix{
{
Metric: labels.FromStrings("__name__", "some_metric", "env", "1"),
Floats: []promql.FPoint{
{T: timestamp.FromTime(baseT.Add(time.Minute)), F: 1},
{T: timestamp.FromTime(baseT.Add(2 * time.Minute)), F: 2},
},
},
{
Metric: labels.FromStrings("__name__", "some_metric", "env", "2"),
Floats: []promql.FPoint{
{T: timestamp.FromTime(baseT.Add(time.Minute)), F: 2},
{T: timestamp.FromTime(baseT.Add(2 * time.Minute)), F: 4},
},
},
{
Metric: labels.FromStrings("__name__", "some_metric", "env", "3"),
Histograms: []promql.HPoint{
{T: timestamp.FromTime(baseT.Add(time.Minute)), H: &histogram.FloatHistogram{Count: 1, CounterResetHint: histogram.NotCounterReset}},
{T: timestamp.FromTime(baseT.Add(2 * time.Minute)), H: &histogram.FloatHistogram{Count: 2, CounterResetHint: histogram.NotCounterReset}},
},
},
},
},
"matches no series": {
expr: "some_nonexistent_metric[1m]",
ts: baseT,
expected: promql.Matrix{},
},
"no samples in range": {
expr: "some_metric[1m]",
ts: baseT.Add(20 * time.Minute),
expected: promql.Matrix{},
},
"metric with stale marker": {
expr: "some_metric_with_stale_marker[3m]",
ts: baseT.Add(3 * time.Minute),
expected: promql.Matrix{
{
Metric: labels.FromStrings("__name__", "some_metric_with_stale_marker"),
Floats: []promql.FPoint{
{T: timestamp.FromTime(baseT.Add(time.Minute)), F: 1},
{T: timestamp.FromTime(baseT.Add(3 * time.Minute)), F: 3},
},
},
},
},
}
for name, testCase := range testCases {
t.Run(name, func(t *testing.T) {
q, err := engine.NewInstantQuery(context.Background(), storage, nil, testCase.expr, testCase.ts)
require.NoError(t, err)
defer q.Close()
res := q.Exec(context.Background())
require.NoError(t, res.Err)
testutil.RequireEqual(t, testCase.expected, res.Value)
})
}
}
func TestQueryLookbackDelta(t *testing.T) {
var (
load = `load 5m

36
promql/promqltest/README.md
View File

@ -106,8 +106,44 @@ eval range from <start> to <end> step <step> <query>
* `<start>` and `<end>` specify the time range of the range query, and use the same syntax as `<time>`
* `<step>` is the step of the range query, and uses the same syntax as `<time>` (eg. `30s`)
* `<expect>`(optional) specifies expected annotations, errors, or result ordering.
* `<expect range vector>` (optional) for an instant query you can specify expected range vector timestamps
* `<expect string> "<string>"` (optional) for matching a string literal
* `<series>` and `<points>` specify the expected values, and follow the same syntax as for `load` above
### `expect string`
This can be used to specify that a string literal is the expected result.
Note that this is only supported on instant queries.
For example;
```
eval instant at 50m ("Foo")
expect string "Foo"
```
The expected string value must be within quotes. Double or back quotes are supported.
### `expect range vector`
This can be used to specify the expected timestamps on a range vector resulting from an instant query.
```
expect range vector <start> to <end> step <step>
```
For example;
```
load 10s
some_metric{env="a"} 1+1x5
some_metric{env="b"} 2+2x5
eval instant at 1m some_metric[1m]
expect range vector from 10s to 1m step 10s
some_metric{env="a"} 2 3 4 5 6
some_metric{env="b"} 4 6 8 10 12
```
### `expect` Syntax
```

157
promql/promqltest/test.go
View File

@ -53,11 +53,14 @@ var (
patEvalRange = regexp.MustCompile(`^eval(?:_(fail|warn|info))?\s+range\s+from\s+(.+)\s+to\s+(.+)\s+step\s+(.+?)\s+(.+)$`)
patExpect = regexp.MustCompile(`^expect\s+(ordered|fail|warn|no_warn|info|no_info)(?:\s+(regex|msg):(.+))?$`)
patMatchAny = regexp.MustCompile(`^.*$`)
patExpectRange = regexp.MustCompile(`^` + rangeVectorPrefix + `\s+from\s+(.+)\s+to\s+(.+)\s+step\s+(.+)$`)
)
const (
defaultEpsilon = 0.000001 // Relative error allowed for sample values.
DefaultMaxSamplesPerQuery = 10000
rangeVectorPrefix = "expect range vector"
expectStringPrefix = "expect string"
)
type TBRun interface {
@ -314,7 +317,58 @@ func validateExpectedCmds(cmd *evalCmd) error {
return nil
}
func (*test) parseEval(lines []string, i int) (int, *evalCmd, error) {
// Given an expected range vector definition, parse the line and return the start & end times and the step duration.
// ie parse a line such as "expect range vector from 10s to 1m step 10s".
// The from and to are parsed as durations and their values added to epoch(0) to form a time.Time.
// The step is parsed as a duration and returned as a time.Duration.
func (t *test) parseExpectRangeVector(line string) (*time.Time, *time.Time, *time.Duration, error) {
parts := patExpectRange.FindStringSubmatch(line)
if len(parts) != 4 {
return nil, nil, nil, fmt.Errorf("invalid range vector definition %q", line)
}
from := parts[1]
to := parts[2]
step := parts[3]
parsedFrom, parsedTo, parsedStep, err := t.parseDurations(from, to, step)
if err != nil {
return nil, nil, nil, err
}
start := testStartTime.Add(time.Duration(*parsedFrom))
end := testStartTime.Add(time.Duration(*parsedTo))
stepDuration := time.Duration(*parsedStep)
return &start, &end, &stepDuration, nil
}
// parseDurations parses the given from, to and step strings to Durations.
// Additionally, a check is performed to ensure to is before from.
func (*test) parseDurations(from, to, step string) (*model.Duration, *model.Duration, *model.Duration, error) {
parsedFrom, err := model.ParseDuration(from)
if err != nil {
return nil, nil, nil, fmt.Errorf("invalid start timestamp definition %q: %w", from, err)
}
parsedTo, err := model.ParseDuration(to)
if err != nil {
return nil, nil, nil, fmt.Errorf("invalid end timestamp definition %q: %w", to, err)
}
if parsedTo < parsedFrom {
return nil, nil, nil, fmt.Errorf("invalid test definition, end timestamp (%s) is before start timestamp (%s)", to, from)
}
parsedStep, err := model.ParseDuration(step)
if err != nil {
return nil, nil, nil, fmt.Errorf("invalid step definition %q: %w", step, err)
}
return &parsedFrom, &parsedTo, &parsedStep, nil
}
func (t *test) parseEval(lines []string, i int) (int, *evalCmd, error) {
instantParts := patEvalInstant.FindStringSubmatch(lines[i])
rangeParts := patEvalRange.FindStringSubmatch(lines[i])
@ -355,10 +409,11 @@ func (*test) parseEval(lines []string, i int) (int, *evalCmd, error) {
}
var cmd *evalCmd
var offset model.Duration
if isInstant {
at := instantParts[2]
offset, err := model.ParseDuration(at)
offset, err = model.ParseDuration(at)
if err != nil {
return i, nil, formatErr("invalid timestamp definition %q: %s", at, err)
}
@ -369,26 +424,12 @@ func (*test) parseEval(lines []string, i int) (int, *evalCmd, error) {
to := rangeParts[3]
step := rangeParts[4]
parsedFrom, err := model.ParseDuration(from)
parsedFrom, parsedTo, parsedStep, err := t.parseDurations(from, to, step)
if err != nil {
return i, nil, formatErr("invalid start timestamp definition %q: %s", from, err)
return i, nil, formatErr(err.Error())
}
parsedTo, err := model.ParseDuration(to)
if err != nil {
return i, nil, formatErr("invalid end timestamp definition %q: %s", to, err)
}
if parsedTo < parsedFrom {
return i, nil, formatErr("invalid test definition, end timestamp (%s) is before start timestamp (%s)", to, from)
}
parsedStep, err := model.ParseDuration(step)
if err != nil {
return i, nil, formatErr("invalid step definition %q: %s", step, err)
}
cmd = newRangeEvalCmd(expr, testStartTime.Add(time.Duration(parsedFrom)), testStartTime.Add(time.Duration(parsedTo)), time.Duration(parsedStep), i+1)
cmd = newRangeEvalCmd(expr, testStartTime.Add(time.Duration(*parsedFrom)), testStartTime.Add(time.Duration(*parsedTo)), time.Duration(*parsedStep), i+1)
}
switch mod {
@ -404,6 +445,8 @@ func (*test) parseEval(lines []string, i int) (int, *evalCmd, error) {
cmd.info = true
}
var expectRangeVector bool
for j := 1; i+1 < len(lines); j++ {
i++
defLine := lines[i]
@ -426,6 +469,32 @@ func (*test) parseEval(lines []string, i int) (int, *evalCmd, error) {
break
}
if strings.HasPrefix(defLine, rangeVectorPrefix) {
start, end, step, err := t.parseExpectRangeVector(defLine)
if err != nil {
return i, nil, formatErr("%w", err)
}
expectRangeVector = true
cmd.start = *start
cmd.end = *end
cmd.step = *step
cmd.eval = *end
cmd.excludeFromRangeQuery = true
continue
}
if strings.HasPrefix(defLine, expectStringPrefix) {
expectString, err := parseAsStringLiteral(defLine)
if err != nil {
return i, nil, formatErr("%w", err)
}
cmd.expectedString = expectString
cmd.excludeFromRangeQuery = true
continue
}
// This would still allow a metric named 'expect' if it is written as 'expect{}'.
if strings.Split(defLine, " ")[0] == "expect" {
annoType, expectedAnno, err := parseExpect(defLine)
@ -450,15 +519,35 @@ func (*test) parseEval(lines []string, i int) (int, *evalCmd, error) {
return i, nil, err
}
// Currently, we are not expecting any matrices.
if len(vals) > 1 && isInstant {
return i, nil, formatErr("expecting multiple values in instant evaluation not allowed")
// Only allow a range vector for an instant query where we have defined the expected range vector timestamps.
if len(vals) > 1 && isInstant && !expectRangeVector {
return i, nil, formatErr("expecting multiple values in instant evaluation not allowed. consider using 'expect range vector' directive to enable a range vector result for an instant query")
}
cmd.expectMetric(j, metric, vals...)
}
return i, cmd, nil
}
// parseAsStringLiteral returns the expected string from an expect string expression.
// It is valid for the line to match the expect string prefix exactly, and an empty string is returned.
func parseAsStringLiteral(line string) (string, error) {
if line == expectStringPrefix {
return "", errors.New("expected string literal not valid - a quoted string literal is required")
}
str := strings.TrimPrefix(line, expectStringPrefix+" ")
if len(str) == 0 {
return "", errors.New("expected string literal not valid - a quoted string literal is required")
}
str, err := strconv.Unquote(str)
if err != nil {
return "", errors.New("expected string literal not valid - check that the string is correctly quoted")
}
return str, nil
}
// getLines returns trimmed lines after removing the comments.
func getLines(input string) []string {
lines := strings.Split(input, "\n")
@ -692,6 +781,7 @@ type evalCmd struct {
end time.Time
step time.Duration
line int
eval time.Time
isRange bool // if false, instant query
fail, warn, ordered, info bool
@ -703,6 +793,12 @@ type evalCmd struct {
metrics map[uint64]labels.Labels
expectScalar bool
expected map[uint64]entry
// we expect a string literal - is set instead of expected
expectedString string
// if true and this is an instant query then we will not test this in a range query scenario
excludeFromRangeQuery bool
}
func (ev *evalCmd) isOrdered() bool {
@ -772,6 +868,7 @@ func newInstantEvalCmd(expr string, start time.Time, line int) *evalCmd {
return &evalCmd{
expr: expr,
start: start,
eval: start,
line: line,
metrics: map[uint64]labels.Labels{},
@ -1016,7 +1113,10 @@ func (ev *evalCmd) compareResult(result parser.Value) error {
if !almost.Equal(exp0.Value, val.V, defaultEpsilon) {
return fmt.Errorf("expected scalar %v but got %v", exp0.Value, val.V)
}
case promql.String:
if ev.expectedString != val.V {
return fmt.Errorf("expected string \"%v\" but got \"%v\"", ev.expectedString, val.V)
}
default:
panic(fmt.Errorf("promql.Test.compareResult: unexpected result type %T", result))
}
@ -1354,11 +1454,12 @@ func (t *test) execRangeEval(cmd *evalCmd, engine promql.QueryEngine) error {
}
func (t *test) execInstantEval(cmd *evalCmd, engine promql.QueryEngine) error {
queries, err := atModifierTestCases(cmd.expr, cmd.start)
queries, err := atModifierTestCases(cmd.expr, cmd.eval)
if err != nil {
return err
}
queries = append([]atModifierTestCase{{expr: cmd.expr, evalTime: cmd.start}}, queries...)
queries = append([]atModifierTestCase{{expr: cmd.expr, evalTime: cmd.eval}}, queries...)
for _, iq := range queries {
if err := t.runInstantQuery(iq, cmd, engine); err != nil {
return err
@ -1395,6 +1496,12 @@ func (t *test) runInstantQuery(iq atModifierTestCase, cmd *evalCmd, engine promq
return fmt.Errorf("error in %s %s (line %d): %w", cmd, iq.expr, cmd.line, err)
}
// this query has have been explicitly excluded from range query testing
// ie it could be that the query result is not an instant vector or scalar
if cmd.excludeFromRangeQuery {
return nil
}
// Check query returns same result in range mode,
// by checking against the middle step.
q, err = engine.NewRangeQuery(t.context, t.storage, nil, iq.expr, iq.evalTime.Add(-time.Minute), iq.evalTime.Add(time.Minute), time.Minute)

138
promql/promqltest/test_test.go
View File

@ -948,6 +948,144 @@ eval instant at 0m http_requests
`,
expectedError: `error in eval http_requests (line 12): invalid expect lines, multiple expect fail lines are not allowed`,
},
"instant query with string literal": {
input: `
eval instant at 50m ("Foo")
expect string "Foo"
`,
},
"instant query with string literal with leading space": {
input: `
eval instant at 50m (" Foo")
expect string " Foo"
`,
},
"instant query with string literal with trailing space": {
input: `
eval instant at 50m ("Foo ")
expect string "Foo "
`,
},
"instant query with string literal as space": {
input: `
eval instant at 50m (" ")
expect string " "
`,
},
"instant query with string literal with empty string": {
input: `
eval instant at 50m ("")
expect string
`,
expectedError: `error in eval ("") (line 3): expected string literal not valid - a quoted string literal is required`,
},
"instant query with string literal with correctly quoted empty string": {
input: `
eval instant at 50m ("")
expect string ""
`,
},
"instant query with string literal - not quoted": {
input: `
eval instant at 50m ("Foo")
expect string Foo
`,
expectedError: `error in eval ("Foo") (line 3): expected string literal not valid - check that the string is correctly quoted`,
},
"instant query with empty string literal": {
input: `
eval instant at 50m ("Foo")
expect string ""
`,
expectedError: `error in eval ("Foo") (line 2): expected string "" but got "Foo"`,
},
"instant query with error string literal": {
input: `
eval instant at 50m ("Foo")
expect string "Bar"
`,
expectedError: `error in eval ("Foo") (line 2): expected string "Bar" but got "Foo"`,
},
"instant query with range result - result does not have a series that is expected": {
input: `
load 10s
some_metric{env="a"} 1+1x5
eval instant at 1m some_metric[1m]
expect range vector from 10s to 1m step 10s
some_metric{env="a"} 2 3 4 5 6
some_metric{env="b"} 4 6 8 10 12
`,
expectedError: `error in eval some_metric[1m] (line 5): expected metric {__name__="some_metric", env="b"} not found`,
},
"instant query with range result - result has a series which is not expected": {
input: `
load 10s
some_metric{env="a"} 1+1x5
some_metric{env="b"} 1+1x5
eval instant at 1m some_metric[1m]
expect range vector from 10s to 1m step 10s
some_metric{env="a"} 2 3 4 5 6
`,
expectedError: `error in eval some_metric[1m] (line 6): unexpected metric {__name__="some_metric", env="b"} in result, has 5 float points [2 @[10000] 3 @[20000] 4 @[30000] 5 @[40000] 6 @[50000]] and 0 histogram points []`,
},
"instant query with range result - result has a value that is not expected": {
input: `
load 10s
some_metric{env="a"} 1+1x5
eval instant at 1m some_metric[1m]
expect range vector from 10s to 1m step 10s
some_metric{env="a"} 9 3 4 5 6
`,
expectedError: `error in eval some_metric[1m] (line 5): expected float value at index 0 (t=10000) for {__name__="some_metric", env="a"} to be 9, but got 2 (result has 5 float points [2 @[10000] 3 @[20000] 4 @[30000] 5 @[40000] 6 @[50000]] and 0 histogram points [])`,
},
"instant query with range result - invalid expect range vector directive": {
input: `
load 10s
some_metric{env="a"} 1+1x5
eval instant at 1m some_metric[1m]
expect range vector from 10s
some_metric{env="a"} 2 3 4 5 6
`,
expectedError: `error in eval some_metric[1m] (line 6): invalid range vector definition "expect range vector from 10s"`,
},
"instant query with range result - result matches expected value": {
input: `
load 1m
some_metric{env="1"} 0+1x4
some_metric{env="2"} 0+2x4
eval instant at 2m some_metric[2m]
expect range vector from 1m to 2m step 60s
some_metric{env="1"} 1 2
some_metric{env="2"} 2 4
`,
},
"instant query with range result - result has a is missing a sample": {
input: `
load 1m
some_metric_with_stale_marker 0 1 stale 3
eval instant at 3m some_metric_with_stale_marker[3m]
expect range vector from 1m to 3m step 60s
some_metric_with_stale_marker{} 1 2 3
`,
expectedError: `error in eval some_metric_with_stale_marker[3m] (line 5): expected 3 float points and 0 histogram points for {__name__="some_metric_with_stale_marker"}, but got 2 float points [1 @[60000] 3 @[180000]] and 0 histogram points []`,
},
"instant query with range result - result has a sample where none is expected": {
input: `
load 1m
some_metric_with_stale_marker 0 1 2 3
eval instant at 3m some_metric_with_stale_marker[3m]
expect range vector from 1m to 3m step 60s
some_metric_with_stale_marker{} 1 _ 3
`,
expectedError: `error in eval some_metric_with_stale_marker[3m] (line 5): expected 2 float points and 0 histogram points for {__name__="some_metric_with_stale_marker"}, but got 3 float points [1 @[60000] 2 @[120000] 3 @[180000]] and 0 histogram points []`,
},
}
for name, testCase := range testCases {

15
promql/promqltest/testdata/literals.test vendored
View File

@ -57,3 +57,18 @@ eval instant at 50m 0 / 0
eval instant at 50m 1 % 0
NaN
eval instant at 50m ("Foo")
expect string `Foo`
eval instant at 50m "Foo"
expect string "Foo"
eval instant at 50m " Foo "
expect string " Foo "
eval instant at 50m ("")
expect string ""
eval instant at 50m ""
expect string ""

34
promql/promqltest/testdata/range_queries.test vendored
View File

@ -71,3 +71,37 @@ eval range from 0 to 2m step 1m requests * 2
{job="1", __address__="bar"} 200 200 200
clear
load 10s
some_metric{env="a"} 1+1x5
some_metric{env="b"} 2+2x5
# Return a range vector - note the use of the expect range vector directive which defines expected range
eval instant at 1m some_metric[1m]
expect range vector from 10s to 1m step 10s
some_metric{env="a"} 2 3 4 5 6
some_metric{env="b"} 4 6 8 10 12
clear
load 1m
some_metric{env="1"} 0+1x4
some_metric{env="2"} 0+2x4
some_metric{env="3"} {{count:0}}+{{count:1}}x4
some_metric_with_stale_marker 0 1 stale 3
eval instant at 2m some_metric[2m]
expect range vector from 1m to 2m step 60s
some_metric{env="1"} 1 2
some_metric{env="2"} 2 4
some_metric{env="3"} {{count:1 counter_reset_hint:not_reset}} {{count:2 counter_reset_hint:not_reset}}
eval instant at 3m some_metric_with_stale_marker[3m]
expect range vector from 1m to 3m step 60s
some_metric_with_stale_marker{} 1 _ 3
eval instant at 1m some_nonexistent_metric[1m]
expect range vector from 10s to 1m step 10s
eval instant at 10m some_metric[1m]
expect range vector from 9m10s to 10m step 1m