mirrors/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2025-08-06 14:17:12 +02:00
Code Issues Projects Releases Wiki Activity

Merge 3b2745bcb9 into 25aee26a57

Browse Source
This commit is contained in:
Andrew Hall 2025-08-05 21:30:52 +01:00 committed by GitHub
commit 20a91fd102
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 267 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
promql/engine.go
View File

@ -1670,12 +1670,8 @@ func (ev *evaluator) eval(ctx context.Context, expr parser.Expr) (parser.Value,
sortedGrouping := e.Grouping
slices.Sort(sortedGrouping)
unwrapParenExpr(&e.Param)
param := unwrapStepInvariantExpr(e.Param)
unwrapParenExpr(&param)
if e.Op == parser.COUNT_VALUES {
valueLabel := param.(*parser.StringLiteral)
valueLabel := e.Param.(*parser.StringLiteral)
if !model.LabelName(valueLabel.Val).IsValid() {
ev.errorf("invalid label name %s", valueLabel)
}
@ -1690,8 +1686,8 @@ func (ev *evaluator) eval(ctx context.Context, expr parser.Expr) (parser.Value,
var warnings annotations.Annotations
originalNumSamples := ev.currentSamples
// param is the number k for topk/bottomk, or q for quantile.
fp, ws := newFParams(ctx, ev, param)
// e.Param is the number k for topk/bottomk, or q for quantile.
fp, ws := newFParams(ctx, ev, e.Param)
warnings.Merge(ws)
// Now fetch the data to be aggregated.
val, ws := ev.eval(ctx, e.Expr)
@ -1711,9 +1707,7 @@ func (ev *evaluator) eval(ctx context.Context, expr parser.Expr) (parser.Value,
// Matrix evaluation always returns the evaluation time,
// so this function needs special handling when given
// a vector selector.
unwrapParenExpr(&e.Args[0])
arg := unwrapStepInvariantExpr(e.Args[0])
unwrapParenExpr(&arg)
vs, ok := arg.(*parser.VectorSelector)
if ok {
return ev.rangeEvalTimestampFunctionOverVectorSelector(ctx, vs, call, e)
@ -1727,9 +1721,7 @@ func (ev *evaluator) eval(ctx context.Context, expr parser.Expr) (parser.Value,
warnings annotations.Annotations
)
for i := range e.Args {
unwrapParenExpr(&e.Args[i])
a := unwrapStepInvariantExpr(e.Args[i])
unwrapParenExpr(&a)
a := e.Args[i]
if _, ok := a.(*parser.MatrixSelector); ok {
matrixArgIndex = i
matrixArg = true
@ -1780,9 +1772,7 @@ func (ev *evaluator) eval(ctx context.Context, expr parser.Expr) (parser.Value,
}
}
unwrapParenExpr(&e.Args[matrixArgIndex])
arg := unwrapStepInvariantExpr(e.Args[matrixArgIndex])
unwrapParenExpr(&arg)
arg := e.Args[matrixArgIndex]
sel := arg.(*parser.MatrixSelector)
selVS := sel.VectorSelector.(*parser.VectorSelector)
@ -3725,8 +3715,8 @@ func unwrapStepInvariantExpr(e parser.Expr) parser.Expr {
}
// PreprocessExpr wraps all possible step invariant parts of the given expression with
// StepInvariantExpr. It also resolves the preprocessors and evaluates duration expressions
// into their numeric values.
// StepInvariantExpr. It also resolves the preprocessors, evaluates duration expressions
// into their numeric values and removes superfluous parenthesis on parameters to functions and aggregations.
func PreprocessExpr(expr parser.Expr, start, end time.Time, step time.Duration) (parser.Expr, error) {
detectHistogramStatsDecoding(expr)
@ -3734,8 +3724,8 @@ func PreprocessExpr(expr parser.Expr, start, end time.Time, step time.Duration)
return nil, err
}
isStepInvariant := preprocessExprHelper(expr, start, end)
if isStepInvariant {
_, shouldWrap := preprocessExprHelper(expr, start, end)
if shouldWrap {
return newStepInvariantExpr(expr), nil
}
return expr, nil
@ -3744,8 +3734,9 @@ func PreprocessExpr(expr parser.Expr, start, end time.Time, step time.Duration)
// preprocessExprHelper wraps the child nodes of the expression
// with a StepInvariantExpr wherever it's step invariant. The returned boolean is true if the
// passed expression qualifies to be wrapped by StepInvariantExpr.
// Also remove superfluous parenthesis on parameters to functions and aggregations.
// It also resolves the preprocessors.
func preprocessExprHelper(expr parser.Expr, start, end time.Time) bool {
func preprocessExprHelper(expr parser.Expr, start, end time.Time) (isStepInvariant, shouldWrap bool) {
switch n := expr.(type) {
case *parser.VectorSelector:
switch n.StartOrEnd {
@ -3754,49 +3745,57 @@ func preprocessExprHelper(expr parser.Expr, start, end time.Time) bool {
case parser.END:
n.Timestamp = makeInt64Pointer(timestamp.FromTime(end))
}
return n.Timestamp != nil
return n.Timestamp != nil, n.Timestamp != nil
case *parser.AggregateExpr:
unwrapParenExpr(&n.Expr)
unwrapParenExpr(&n.Param)
return preprocessExprHelper(n.Expr, start, end)
case *parser.BinaryExpr:
isInvariant1, isInvariant2 := preprocessExprHelper(n.LHS, start, end), preprocessExprHelper(n.RHS, start, end)
isInvariant1, shouldWrap1 := preprocessExprHelper(n.LHS, start, end)
isInvariant2, shouldWrap2 := preprocessExprHelper(n.RHS, start, end)
if isInvariant1 && isInvariant2 {
return true
return true, true
}
if isInvariant1 {
if shouldWrap1 {
n.LHS = newStepInvariantExpr(n.LHS)
}
if isInvariant2 {
if shouldWrap2 {
n.RHS = newStepInvariantExpr(n.RHS)
}
return false
return false, false
case *parser.Call:
_, ok := AtModifierUnsafeFunctions[n.Func.Name]
isStepInvariant := !ok
isStepInvariantSlice := make([]bool, len(n.Args))
shouldWrap := make([]bool, len(n.Args))
for i := range n.Args {
isStepInvariantSlice[i] = preprocessExprHelper(n.Args[i], start, end)
isStepInvariant = isStepInvariant && isStepInvariantSlice[i]
unwrapParenExpr(&n.Args[i])
var argIsStepInvariant bool
argIsStepInvariant, shouldWrap[i] = preprocessExprHelper(n.Args[i], start, end)
isStepInvariant = isStepInvariant && argIsStepInvariant
}
if isStepInvariant {
// The function and all arguments are step invariant.
return true
return true, true
}
for i, isi := range isStepInvariantSlice {
for i, isi := range shouldWrap {
if isi {
n.Args[i] = newStepInvariantExpr(n.Args[i])
}
}
return false
return false, false
case *parser.MatrixSelector:
return preprocessExprHelper(n.VectorSelector, start, end)
// We don't need to wrap a MatrixSelector because functions over range vectors evaluate those directly,
// and they can't appear at top level in a range query.
isStepInvariant, _ := preprocessExprHelper(n.VectorSelector, start, end)
return isStepInvariant, false
case *parser.SubqueryExpr:
// Since we adjust offset for the @ modifier evaluation,
@ -3804,7 +3803,7 @@ func preprocessExprHelper(expr parser.Expr, start, end time.Time) bool {
// Hence we wrap the inside of subquery irrespective of
// @ on subquery (given it is also step invariant) so that
// it is evaluated only once w.r.t. the start time of subquery.
isInvariant := preprocessExprHelper(n.Expr, start, end)
isInvariant, _ := preprocessExprHelper(n.Expr, start, end)
if isInvariant {
n.Expr = newStepInvariantExpr(n.Expr)
}
@ -3814,7 +3813,7 @@ func preprocessExprHelper(expr parser.Expr, start, end time.Time) bool {
case parser.END:
n.Timestamp = makeInt64Pointer(timestamp.FromTime(end))
}
return n.Timestamp != nil
return n.Timestamp != nil, n.Timestamp != nil
case *parser.ParenExpr:
return preprocessExprHelper(n.Expr, start, end)
@ -3823,7 +3822,7 @@ func preprocessExprHelper(expr parser.Expr, start, end time.Time) bool {
return preprocessExprHelper(n.Expr, start, end)
case *parser.StringLiteral, *parser.NumberLiteral:
return true
return true, false
}
panic(fmt.Sprintf("found unexpected node %#v", expr))

110
promql/engine_test.go
View File

@ -2304,20 +2304,16 @@ func TestPreprocessAndWrapWithStepInvariantExpr(t *testing.T) {
}{
{
input: "123.4567",
expected: &parser.StepInvariantExpr{
Expr: &parser.NumberLiteral{
Val: 123.4567,
PosRange: posrange.PositionRange{Start: 0, End: 8},
},
expected: &parser.NumberLiteral{
Val: 123.4567,
PosRange: posrange.PositionRange{Start: 0, End: 8},
},
},
{
input: `"foo"`,
expected: &parser.StepInvariantExpr{
Expr: &parser.StringLiteral{
Val: "foo",
PosRange: posrange.PositionRange{Start: 0, End: 5},
},
expected: &parser.StringLiteral{
Val: "foo",
PosRange: posrange.PositionRange{Start: 0, End: 5},
},
},
{
@ -2427,23 +2423,21 @@ func TestPreprocessAndWrapWithStepInvariantExpr(t *testing.T) {
},
{
input: `test{a="b"}[5y] @ 1603774699`,
expected: &parser.StepInvariantExpr{
Expr: &parser.MatrixSelector{
VectorSelector: &parser.VectorSelector{
Name: "test",
Timestamp: makeInt64Pointer(1603774699000),
LabelMatchers: []*labels.Matcher{
parser.MustLabelMatcher(labels.MatchEqual, "a", "b"),
parser.MustLabelMatcher(labels.MatchEqual, "__name__", "test"),
},
PosRange: posrange.PositionRange{
Start: 0,
End: 11,
},
expected: &parser.MatrixSelector{
VectorSelector: &parser.VectorSelector{
Name: "test",
Timestamp: makeInt64Pointer(1603774699000),
LabelMatchers: []*labels.Matcher{
parser.MustLabelMatcher(labels.MatchEqual, "a", "b"),
parser.MustLabelMatcher(labels.MatchEqual, "__name__", "test"),
},
PosRange: posrange.PositionRange{
Start: 0,
End: 11,
},
Range: 5 * 365 * 24 * time.Hour,
EndPos: 28,
},
Range: 5 * 365 * 24 * time.Hour,
EndPos: 28,
},
},
{
@ -2942,45 +2936,53 @@ func TestPreprocessAndWrapWithStepInvariantExpr(t *testing.T) {
},
},
{
input: `test[5y] @ start()`,
input: `sum_over_time(test[5y] @ start())`,
expected: &parser.StepInvariantExpr{
Expr: &parser.MatrixSelector{
VectorSelector: &parser.VectorSelector{
Name: "test",
Timestamp: makeInt64Pointer(timestamp.FromTime(startTime)),
StartOrEnd: parser.START,
LabelMatchers: []*labels.Matcher{
parser.MustLabelMatcher(labels.MatchEqual, "__name__", "test"),
},
PosRange: posrange.PositionRange{
Start: 0,
End: 4,
Expr: &parser.Call{
Func: &parser.Function{
Name: "sum_over_time",
ArgTypes: []parser.ValueType{parser.ValueTypeMatrix},
ReturnType: parser.ValueTypeVector,
},
Args: parser.Expressions{
&parser.MatrixSelector{
VectorSelector: &parser.VectorSelector{
Name: "test",
Timestamp: makeInt64Pointer(timestamp.FromTime(startTime)),
StartOrEnd: parser.START,
LabelMatchers: []*labels.Matcher{
parser.MustLabelMatcher(labels.MatchEqual, "__name__", "test"),
},
PosRange: posrange.PositionRange{
Start: 14,
End: 18,
},
},
Range: 5 * 365 * 24 * time.Hour,
EndPos: 32,
},
},
Range: 5 * 365 * 24 * time.Hour,
EndPos: 18,
PosRange: posrange.PositionRange{Start: 0, End: 32},
},
},
},
{
input: `test[5y] @ end()`,
expected: &parser.StepInvariantExpr{
Expr: &parser.MatrixSelector{
VectorSelector: &parser.VectorSelector{
Name: "test",
Timestamp: makeInt64Pointer(timestamp.FromTime(endTime)),
StartOrEnd: parser.END,
LabelMatchers: []*labels.Matcher{
parser.MustLabelMatcher(labels.MatchEqual, "__name__", "test"),
},
PosRange: posrange.PositionRange{
Start: 0,
End: 4,
},
expected: &parser.MatrixSelector{
VectorSelector: &parser.VectorSelector{
Name: "test",
Timestamp: makeInt64Pointer(timestamp.FromTime(endTime)),
StartOrEnd: parser.END,
LabelMatchers: []*labels.Matcher{
parser.MustLabelMatcher(labels.MatchEqual, "__name__", "test"),
},
PosRange: posrange.PositionRange{
Start: 0,
End: 4,
},
Range: 5 * 365 * 24 * time.Hour,
EndPos: 16,
},
Range: 5 * 365 * 24 * time.Hour,
EndPos: 16,
},
},
{

4
promql/functions.go
View File

@ -1946,9 +1946,7 @@ func createLabelsForAbsentFunction(expr parser.Expr) labels.Labels {
}
func stringFromArg(e parser.Expr) string {
tmp := unwrapStepInvariantExpr(e) // Unwrap StepInvariant
unwrapParenExpr(&tmp) // Optionally unwrap ParenExpr
return tmp.(*parser.StringLiteral).Val
return e.(*parser.StringLiteral).Val
}
func stringSliceFromArgs(args parser.Expressions) []string {

32
promql/promqltest/README.md
View File

@ -106,8 +106,40 @@ eval range from <start> to <end> step <step> <query>
* `<start>` and `<end>` specify the time range of the range query, and use the same syntax as `<time>`
* `<step>` is the step of the range query, and uses the same syntax as `<time>` (eg. `30s`)
* `<expect>`(optional) specifies expected annotations, errors, or result ordering.
* `<expect range vector>` (optional) for an instant query you can specify expected range vector timestamps
* `<series>` and `<points>` specify the expected values, and follow the same syntax as for `load` above
Note - a string literal instant series is now suppported. The string literal can be defined by placing the string within quotes ("").
This is only valid for an instant query.
For instance;
```
eval instant at 50m ("Foo")
"Foo"
```
### `expect range vector`
This can be used to specify the expected timestamps on a range vector resulting from an instant query.
```
expect range vector <start> to <end> step <step>
```
For example;
```
load 10s
some_metric{env="a"} 1+1x5
some_metric{env="b"} 2+2x5
eval instant at 1m some_metric[1m]
expect range vector from 10s to 1m step 10s
some_metric{env="a"} 2 3 4 5 6
some_metric{env="b"} 4 6 8 10 12
```
### `expect` Syntax
```

126
promql/promqltest/test.go
View File

@ -46,6 +46,10 @@ import (
"github.com/prometheus/prometheus/util/testutil"
)
const (
rangeVectorPrefix = "expect range vector"
)
var (
patSpace = regexp.MustCompile("[\t ]+")
patLoad = regexp.MustCompile(`^load(?:_(with_nhcb))?\s+(.+?)$`)
@ -53,6 +57,7 @@ var (
patEvalRange = regexp.MustCompile(`^eval(?:_(fail|warn|info))?\s+range\s+from\s+(.+)\s+to\s+(.+)\s+step\s+(.+?)\s+(.+)$`)
patExpect = regexp.MustCompile(`^expect\s+(ordered|fail|warn|no_warn|info|no_info)(?:\s+(regex|msg):(.+))?$`)
patMatchAny = regexp.MustCompile(`^.*$`)
patExpectRange = regexp.MustCompile(`^` + rangeVectorPrefix + `\s+from\s+(.+)\s+to\s+(.+)\s+step\s+(.+)$`)
)
const (
@ -314,6 +319,46 @@ func validateExpectedCmds(cmd *evalCmd) error {
return nil
}
// Given an expected range vector definition, parse the line and return the start & end times and the step duration.
// ie parse a line such as "expect range vector from 10s to 1m step 10s".
// The from and to are parsed as durations and their values added to epoch(0) to form a time.Time.
// The step is parsed as a duration and returned as a time.Duration.
func (t *test) parseExpectRangeVector(line string) (*time.Time, *time.Time, *time.Duration, error) {
parts := patExpectRange.FindStringSubmatch(line)
if len(parts) != 4 {
return nil, nil, nil, fmt.Errorf("invalid range vector definition %q", line)
}
from := parts[1]
to := parts[2]
step := parts[3]
parsedFrom, err := model.ParseDuration(from)
if err != nil {
return nil, nil, nil, fmt.Errorf("invalid range vector start timestamp offset definition %q: %w", from, err)
}
parsedTo, err := model.ParseDuration(to)
if err != nil {
return nil, nil, nil, fmt.Errorf("invalid range vector end timestamp offset definition %q: %w", to, err)
}
if parsedTo < parsedFrom {
return nil, nil, nil, fmt.Errorf("invalid range vector timestamp offsets, end timestamp (%s) is before start timestamp (%s)", to, from)
}
parsedStep, err := model.ParseDuration(step)
if err != nil {
return nil, nil, nil, fmt.Errorf("invalid range vector step definition %q: %w", step, err)
}
start := time.Unix(0, 0).Add(time.Duration(parsedFrom))
end := time.Unix(0, 0).Add(time.Duration(parsedTo))
stepDuration := time.Duration(parsedStep)
return &start, &end, &stepDuration, nil
}
func (t *test) parseEval(lines []string, i int) (int, *evalCmd, error) {
instantParts := patEvalInstant.FindStringSubmatch(lines[i])
rangeParts := patEvalRange.FindStringSubmatch(lines[i])
@ -404,6 +449,8 @@ func (t *test) parseEval(lines []string, i int) (int, *evalCmd, error) {
cmd.info = true
}
var allowExpectedRangeVector bool
for j := 1; i+1 < len(lines); j++ {
i++
defLine := lines[i]
@ -426,6 +473,20 @@ func (t *test) parseEval(lines []string, i int) (int, *evalCmd, error) {
break
}
if strings.HasPrefix(defLine, rangeVectorPrefix) {
start, end, step, err := t.parseExpectRangeVector(defLine)
if err != nil {
return i, nil, formatErr("%w", err)
}
allowExpectedRangeVector = true
cmd.start = *start
cmd.end = *end
cmd.step = *step
cmd.excludeFromRangeQuery = true
cmd.excludeFromAtModifier = true
continue
}
// This would still allow a metric named 'expect' if it is written as 'expect{}'.
if strings.Split(defLine, " ")[0] == "expect" {
annoType, expectedAnno, err := parseExpect(defLine)
@ -447,11 +508,18 @@ func (t *test) parseEval(lines []string, i int) (int, *evalCmd, error) {
}
metric, vals, err := parseSeries(defLine, i)
if err != nil {
literal, err1 := parseAsStringLiteral(defLine)
if err1 == nil {
cmd.expectedString = literal
cmd.excludeFromRangeQuery = true
return i, cmd, nil
}
return i, nil, err
}
// Currently, we are not expecting any matrices.
if len(vals) > 1 && isInstant {
// Currently, we only allow a range vector for an instant query is where we have defined the expected range vector timestamps.
if len(vals) > 1 && isInstant && !allowExpectedRangeVector {
return i, nil, formatErr("expecting multiple values in instant evaluation not allowed")
}
cmd.expectMetric(j, metric, vals...)
@ -459,6 +527,16 @@ func (t *test) parseEval(lines []string, i int) (int, *evalCmd, error) {
return i, cmd, nil
}
// Parse a string literal from the given input
// The literal must be enclosed in double quotes - the returned value is simply the input with the first and last quotes removed
// No further validation is performed on the literal - ie we don't check for unbalanced quotes, escaped quotes, etc.
func parseAsStringLiteral(input string) (string, error) {
if strings.HasPrefix(input, `"`) && strings.HasSuffix(input, `"`) {
return input[1 : len(input)-1], nil
}
return "", fmt.Errorf("invalid string literal: %s", input)
}
// getLines returns trimmed lines after removing the comments.
func getLines(input string) []string {
lines := strings.Split(input, "\n")
@ -703,6 +781,15 @@ type evalCmd struct {
metrics map[uint64]labels.Labels
expectScalar bool
expected map[uint64]entry
// we expect a string literal - is set instead of expected
expectedString string
// if true and this is an instant query then we will not test this in a range query scenario
excludeFromRangeQuery bool
// if true and this is an instant query then we will exclude it from adding additional at modifier test cases
excludeFromAtModifier bool
}
func (ev *evalCmd) isOrdered() bool {
@ -777,6 +864,9 @@ func newInstantEvalCmd(expr string, start time.Time, line int) *evalCmd {
metrics: map[uint64]labels.Labels{},
expected: map[uint64]entry{},
expectedCmds: map[expectCmdType][]expectCmd{},
excludeFromRangeQuery: false,
excludeFromAtModifier: false,
}
}
@ -1016,7 +1106,10 @@ func (ev *evalCmd) compareResult(result parser.Value) error {
if !almost.Equal(exp0.Value, val.V, defaultEpsilon) {
return fmt.Errorf("expected scalar %v but got %v", exp0.Value, val.V)
}
case promql.String:
if ev.expectedString != val.V {
return fmt.Errorf("expected string %v but got %v", ev.expectedString, val.V)
}
default:
panic(fmt.Errorf("promql.Test.compareResult: unexpected result type %T", result))
}
@ -1354,11 +1447,24 @@ func (t *test) execRangeEval(cmd *evalCmd, engine promql.QueryEngine) error {
}
func (t *test) execInstantEval(cmd *evalCmd, engine promql.QueryEngine) error {
queries, err := atModifierTestCases(cmd.expr, cmd.start)
if err != nil {
return err
var queries []atModifierTestCase
var err error
if !cmd.excludeFromAtModifier {
queries, err = atModifierTestCases(cmd.expr, cmd.start)
if err != nil {
return err
}
}
queries = append([]atModifierTestCase{{expr: cmd.expr, evalTime: cmd.start}}, queries...)
// by default we evaluate the query at the start time
// but if we are expecting a range vector ie metric[5m] then we evaluate the query at the end time
evalTime := cmd.start
if cmd.end.After(cmd.start) {
evalTime = cmd.end
}
queries = append([]atModifierTestCase{{expr: cmd.expr, evalTime: evalTime}}, queries...)
for _, iq := range queries {
if err := t.runInstantQuery(iq, cmd, engine); err != nil {
return err
@ -1395,6 +1501,12 @@ func (t *test) runInstantQuery(iq atModifierTestCase, cmd *evalCmd, engine promq
return fmt.Errorf("error in %s %s (line %d): %w", cmd, iq.expr, cmd.line, err)
}
// this query has have been explicitly excluded from range query testing
// ie it could be that the query result is not an instant vector or scalar
if cmd.excludeFromRangeQuery {
return nil
}
// Check query returns same result in range mode,
// by checking against the middle step.
q, err = engine.NewRangeQuery(t.context, t.storage, nil, iq.expr, iq.evalTime.Add(-time.Minute), iq.evalTime.Add(time.Minute), time.Minute)

12
promql/promqltest/testdata/literals.test vendored
View File

@ -57,3 +57,15 @@ eval instant at 50m 0 / 0
eval instant at 50m 1 % 0
NaN
eval instant at 50m ("Foo")
"Foo"
eval instant at 50m "Foo"
"Foo"
eval instant at 50m ("")
""
eval instant at 50m ""
""

12
promql/promqltest/testdata/range_queries.test vendored
View File

@ -71,3 +71,15 @@ eval range from 0 to 2m step 1m requests * 2
{job="1", __address__="bar"} 200 200 200
clear
load 10s
some_metric{env="a"} 1+1x5
some_metric{env="b"} 2+2x5
# Return a range vector - note that we use the range vector to define our expected timestamps in the result vectors
eval instant at 1m some_metric[1m]
expect range vector from 10s to 1m step 10s
some_metric{env="a"} 2 3 4 5 6
some_metric{env="b"} 4 6 8 10 12
clear