mirrors/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2025-09-21 13:51:00 +02:00
Code Issues Projects Releases Wiki Activity

Merge pull request #16896 from bboreham/wrap-less

Browse Source 
[PERF] PromQL: Move more work to preprocessing step
This commit is contained in:
Bryan Boreham 2025-08-13 14:17:30 +01:00 committed by GitHub
commit 8fff489c53
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 97 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
promql/engine.go
View File

@ -1670,12 +1670,8 @@ func (ev *evaluator) eval(ctx context.Context, expr parser.Expr) (parser.Value,
sortedGrouping := e.Grouping
slices.Sort(sortedGrouping)
unwrapParenExpr(&e.Param)
param := unwrapStepInvariantExpr(e.Param)
unwrapParenExpr(&param)
if e.Op == parser.COUNT_VALUES {
valueLabel := param.(*parser.StringLiteral)
valueLabel := e.Param.(*parser.StringLiteral)
if !model.LabelName(valueLabel.Val).IsValid() {
ev.errorf("invalid label name %s", valueLabel)
}
@ -1690,8 +1686,8 @@ func (ev *evaluator) eval(ctx context.Context, expr parser.Expr) (parser.Value,
var warnings annotations.Annotations
originalNumSamples := ev.currentSamples
// param is the number k for topk/bottomk, or q for quantile.
fp, ws := newFParams(ctx, ev, param)
// e.Param is the number k for topk/bottomk, or q for quantile.
fp, ws := newFParams(ctx, ev, e.Param)
warnings.Merge(ws)
// Now fetch the data to be aggregated.
val, ws := ev.eval(ctx, e.Expr)
@ -1711,9 +1707,7 @@ func (ev *evaluator) eval(ctx context.Context, expr parser.Expr) (parser.Value,
// Matrix evaluation always returns the evaluation time,
// so this function needs special handling when given
// a vector selector.
unwrapParenExpr(&e.Args[0])
arg := unwrapStepInvariantExpr(e.Args[0])
unwrapParenExpr(&arg)
vs, ok := arg.(*parser.VectorSelector)
if ok {
return ev.rangeEvalTimestampFunctionOverVectorSelector(ctx, vs, call, e)
@ -1727,9 +1721,7 @@ func (ev *evaluator) eval(ctx context.Context, expr parser.Expr) (parser.Value,
warnings annotations.Annotations
)
for i := range e.Args {
unwrapParenExpr(&e.Args[i])
a := unwrapStepInvariantExpr(e.Args[i])
unwrapParenExpr(&a)
a := e.Args[i]
if _, ok := a.(*parser.MatrixSelector); ok {
matrixArgIndex = i
matrixArg = true
@ -1780,9 +1772,7 @@ func (ev *evaluator) eval(ctx context.Context, expr parser.Expr) (parser.Value,
}
}
unwrapParenExpr(&e.Args[matrixArgIndex])
arg := unwrapStepInvariantExpr(e.Args[matrixArgIndex])
unwrapParenExpr(&arg)
arg := e.Args[matrixArgIndex]
sel := arg.(*parser.MatrixSelector)
selVS := sel.VectorSelector.(*parser.VectorSelector)
@ -2108,11 +2098,6 @@ func (ev *evaluator) eval(ctx context.Context, expr parser.Expr) (parser.Value,
ev.samplesStats.IncrementSamplesAtTimestamp(ev.endTimestamp, newEv.samplesStats.TotalSamples)
return res, ws
case *parser.StepInvariantExpr:
switch ce := e.Expr.(type) {
case *parser.StringLiteral, *parser.NumberLiteral:
return ev.eval(ctx, ce)
}
newEv := &evaluator{
startTimestamp: ev.startTimestamp,
endTimestamp: ev.startTimestamp, // Always a single evaluation.
@ -3725,8 +3710,8 @@ func unwrapStepInvariantExpr(e parser.Expr) parser.Expr {
}
// PreprocessExpr wraps all possible step invariant parts of the given expression with
// StepInvariantExpr. It also resolves the preprocessors and evaluates duration expressions
// into their numeric values.
// StepInvariantExpr. It also resolves the preprocessors, evaluates duration expressions
// into their numeric values and removes superfluous parenthesis on parameters to functions and aggregations.
func PreprocessExpr(expr parser.Expr, start, end time.Time, step time.Duration) (parser.Expr, error) {
detectHistogramStatsDecoding(expr)
@ -3734,18 +3719,20 @@ func PreprocessExpr(expr parser.Expr, start, end time.Time, step time.Duration)
return nil, err
}
isStepInvariant := preprocessExprHelper(expr, start, end)
if isStepInvariant {
_, shouldWrap := preprocessExprHelper(expr, start, end)
if shouldWrap {
return newStepInvariantExpr(expr), nil
}
return expr, nil
}
// preprocessExprHelper wraps the child nodes of the expression
// with a StepInvariantExpr wherever it's step invariant. The returned boolean is true if the
// passed expression qualifies to be wrapped by StepInvariantExpr.
// It also resolves the preprocessors.
func preprocessExprHelper(expr parser.Expr, start, end time.Time) bool {
// preprocessExprHelper wraps child nodes of expr with a StepInvariantExpr,
// at the highest level within the tree that is step-invariant.
// Also resolves start() and end() on selector and subquery nodes.
// Also remove superfluous parenthesis on parameters to functions and aggregations.
// Return isStepInvariant is true when the whole subexpression is step invariant.
// Return shoudlWrap is false for cases like MatrixSelector and StringLiteral that never need to be wrapped.
func preprocessExprHelper(expr parser.Expr, start, end time.Time) (isStepInvariant, shouldWrap bool) {
switch n := expr.(type) {
case *parser.VectorSelector:
switch n.StartOrEnd {
@ -3754,49 +3741,56 @@ func preprocessExprHelper(expr parser.Expr, start, end time.Time) bool {
case parser.END:
n.Timestamp = makeInt64Pointer(timestamp.FromTime(end))
}
return n.Timestamp != nil
return n.Timestamp != nil, n.Timestamp != nil
case *parser.AggregateExpr:
unwrapParenExpr(&n.Expr)
unwrapParenExpr(&n.Param)
return preprocessExprHelper(n.Expr, start, end)
case *parser.BinaryExpr:
isInvariant1, isInvariant2 := preprocessExprHelper(n.LHS, start, end), preprocessExprHelper(n.RHS, start, end)
if isInvariant1 && isInvariant2 {
return true
isInvariantLHS, shouldWrapLHS := preprocessExprHelper(n.LHS, start, end)
isInvariantRHS, shouldWrapRHS := preprocessExprHelper(n.RHS, start, end)
if isInvariantLHS && isInvariantRHS {
return true, true
}
if isInvariant1 {
if shouldWrapLHS {
n.LHS = newStepInvariantExpr(n.LHS)
}
if isInvariant2 {
if shouldWrapRHS {
n.RHS = newStepInvariantExpr(n.RHS)
}
return false
return false, false
case *parser.Call:
_, ok := AtModifierUnsafeFunctions[n.Func.Name]
isStepInvariant := !ok
isStepInvariantSlice := make([]bool, len(n.Args))
shouldWrap := make([]bool, len(n.Args))
for i := range n.Args {
isStepInvariantSlice[i] = preprocessExprHelper(n.Args[i], start, end)
isStepInvariant = isStepInvariant && isStepInvariantSlice[i]
unwrapParenExpr(&n.Args[i])
var argIsStepInvariant bool
argIsStepInvariant, shouldWrap[i] = preprocessExprHelper(n.Args[i], start, end)
isStepInvariant = isStepInvariant && argIsStepInvariant
}
if isStepInvariant {
// The function and all arguments are step invariant.
return true
return true, true
}
for i, isi := range isStepInvariantSlice {
for i, isi := range shouldWrap {
if isi {
n.Args[i] = newStepInvariantExpr(n.Args[i])
}
}
return false
return false, false
case *parser.MatrixSelector:
return preprocessExprHelper(n.VectorSelector, start, end)
// We don't need to wrap a MatrixSelector because functions over range vectors evaluate those directly.
isStepInvariant, _ := preprocessExprHelper(n.VectorSelector, start, end)
return isStepInvariant, false
case *parser.SubqueryExpr:
// Since we adjust offset for the @ modifier evaluation,
@ -3804,7 +3798,7 @@ func preprocessExprHelper(expr parser.Expr, start, end time.Time) bool {
// Hence we wrap the inside of subquery irrespective of
// @ on subquery (given it is also step invariant) so that
// it is evaluated only once w.r.t. the start time of subquery.
isInvariant := preprocessExprHelper(n.Expr, start, end)
isInvariant, _ := preprocessExprHelper(n.Expr, start, end)
if isInvariant {
n.Expr = newStepInvariantExpr(n.Expr)
}
@ -3814,7 +3808,7 @@ func preprocessExprHelper(expr parser.Expr, start, end time.Time) bool {
case parser.END:
n.Timestamp = makeInt64Pointer(timestamp.FromTime(end))
}
return n.Timestamp != nil
return n.Timestamp != nil, n.Timestamp != nil
case *parser.ParenExpr:
return preprocessExprHelper(n.Expr, start, end)
@ -3823,7 +3817,7 @@ func preprocessExprHelper(expr parser.Expr, start, end time.Time) bool {
return preprocessExprHelper(n.Expr, start, end)
case *parser.StringLiteral, *parser.NumberLiteral:
return true
return true, false
}
panic(fmt.Sprintf("found unexpected node %#v", expr))

110
promql/engine_test.go
View File

@ -2304,20 +2304,16 @@ func TestPreprocessAndWrapWithStepInvariantExpr(t *testing.T) {
}{
{
input: "123.4567",
expected: &parser.StepInvariantExpr{
Expr: &parser.NumberLiteral{
Val: 123.4567,
PosRange: posrange.PositionRange{Start: 0, End: 8},
},
expected: &parser.NumberLiteral{
Val: 123.4567,
PosRange: posrange.PositionRange{Start: 0, End: 8},
},
},
{
input: `"foo"`,
expected: &parser.StepInvariantExpr{
Expr: &parser.StringLiteral{
Val: "foo",
PosRange: posrange.PositionRange{Start: 0, End: 5},
},
expected: &parser.StringLiteral{
Val: "foo",
PosRange: posrange.PositionRange{Start: 0, End: 5},
},
},
{
@ -2427,23 +2423,21 @@ func TestPreprocessAndWrapWithStepInvariantExpr(t *testing.T) {
},
{
input: `test{a="b"}[5y] @ 1603774699`,
expected: &parser.StepInvariantExpr{
Expr: &parser.MatrixSelector{
VectorSelector: &parser.VectorSelector{
Name: "test",
Timestamp: makeInt64Pointer(1603774699000),
LabelMatchers: []*labels.Matcher{
parser.MustLabelMatcher(labels.MatchEqual, "a", "b"),
parser.MustLabelMatcher(labels.MatchEqual, "__name__", "test"),
},
PosRange: posrange.PositionRange{
Start: 0,
End: 11,
},
expected: &parser.MatrixSelector{
VectorSelector: &parser.VectorSelector{
Name: "test",
Timestamp: makeInt64Pointer(1603774699000),
LabelMatchers: []*labels.Matcher{
parser.MustLabelMatcher(labels.MatchEqual, "a", "b"),
parser.MustLabelMatcher(labels.MatchEqual, "__name__", "test"),
},
PosRange: posrange.PositionRange{
Start: 0,
End: 11,
},
Range: 5 * 365 * 24 * time.Hour,
EndPos: 28,
},
Range: 5 * 365 * 24 * time.Hour,
EndPos: 28,
},
},
{
@ -2942,45 +2936,53 @@ func TestPreprocessAndWrapWithStepInvariantExpr(t *testing.T) {
},
},
{
input: `test[5y] @ start()`,
input: `sum_over_time(test[5y] @ start())`,
expected: &parser.StepInvariantExpr{
Expr: &parser.MatrixSelector{
VectorSelector: &parser.VectorSelector{
Name: "test",
Timestamp: makeInt64Pointer(timestamp.FromTime(startTime)),
StartOrEnd: parser.START,
LabelMatchers: []*labels.Matcher{
parser.MustLabelMatcher(labels.MatchEqual, "__name__", "test"),
},
PosRange: posrange.PositionRange{
Start: 0,
End: 4,
Expr: &parser.Call{
Func: &parser.Function{
Name: "sum_over_time",
ArgTypes: []parser.ValueType{parser.ValueTypeMatrix},
ReturnType: parser.ValueTypeVector,
},
Args: parser.Expressions{
&parser.MatrixSelector{
VectorSelector: &parser.VectorSelector{
Name: "test",
Timestamp: makeInt64Pointer(timestamp.FromTime(startTime)),
StartOrEnd: parser.START,
LabelMatchers: []*labels.Matcher{
parser.MustLabelMatcher(labels.MatchEqual, "__name__", "test"),
},
PosRange: posrange.PositionRange{
Start: 14,
End: 18,
},
},
Range: 5 * 365 * 24 * time.Hour,
EndPos: 32,
},
},
Range: 5 * 365 * 24 * time.Hour,
EndPos: 18,
PosRange: posrange.PositionRange{Start: 0, End: 33},
},
},
},
{
input: `test[5y] @ end()`,
expected: &parser.StepInvariantExpr{
Expr: &parser.MatrixSelector{
VectorSelector: &parser.VectorSelector{
Name: "test",
Timestamp: makeInt64Pointer(timestamp.FromTime(endTime)),
StartOrEnd: parser.END,
LabelMatchers: []*labels.Matcher{
parser.MustLabelMatcher(labels.MatchEqual, "__name__", "test"),
},
PosRange: posrange.PositionRange{
Start: 0,
End: 4,
},
expected: &parser.MatrixSelector{
VectorSelector: &parser.VectorSelector{
Name: "test",
Timestamp: makeInt64Pointer(timestamp.FromTime(endTime)),
StartOrEnd: parser.END,
LabelMatchers: []*labels.Matcher{
parser.MustLabelMatcher(labels.MatchEqual, "__name__", "test"),
},
PosRange: posrange.PositionRange{
Start: 0,
End: 4,
},
Range: 5 * 365 * 24 * time.Hour,
EndPos: 16,
},
Range: 5 * 365 * 24 * time.Hour,
EndPos: 16,
},
},
{

4
promql/functions.go
View File

@ -1946,9 +1946,7 @@ func createLabelsForAbsentFunction(expr parser.Expr) labels.Labels {
}
func stringFromArg(e parser.Expr) string {
tmp := unwrapStepInvariantExpr(e) // Unwrap StepInvariant
unwrapParenExpr(&tmp) // Optionally unwrap ParenExpr
return tmp.(*parser.StringLiteral).Val
return e.(*parser.StringLiteral).Val
}
func stringSliceFromArgs(args parser.Expressions) []string {