proc: expose breakpoint hitcounts in expressions (#3874)

Expose breakpoint hitcounts in the expression language through the
special variable runtime.bphitcount:
  delve.bphitcount[1]
  delve.bphitcount["bpname"]
will evaluate respectively to the hitcount of breakpoint with id == 1
and to the hitcount of the breakpoint named "bpname".

This is intended to be used in breakpoint conditions and allows
breakpoints to be chained such that one breakpoint is only hit after a
different is hit first.

A few optimizations are implemented so that chained breakpoints are
evaluated efficiently.
This commit is contained in:
Alessandro Arzilli 2025-03-05 21:39:47 +01:00 committed by GitHub
parent 2685a42bc0
commit 6ef45f534c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 461 additions and 29 deletions

View File

@ -211,7 +211,7 @@ Set breakpoint condition.
Specifies that the breakpoint, tracepoint or watchpoint should break only if the boolean expression is true.
See [Documentation/cli/expr.md](//github.com/go-delve/delve/tree/master/Documentation/cli/expr.md) for a description of supported expressions.
See [Documentation/cli/expr.md](//github.com/go-delve/delve/tree/master/Documentation/cli/expr.md) for a description of supported expressions and [Documentation/cli/cond.md](//github.com/go-delve/delve/tree/master/Documentation/cli/cond.md) for a description of how breakpoint conditions are evaluated.
With the -hitcount option a condition on the breakpoint hit count can be set, the following operators are supported

15
Documentation/cli/cond.md Normal file
View File

@ -0,0 +1,15 @@
# Breakpoint conditions
Breakpoints have two conditions:
* The normal condition, which is specified using the command `cond <breakpoint> <expr>` (or by setting the Cond field when amending a breakpoint via the API), is any [expression](expr.md) which evaluates to true or false.
* The hitcount condition, which is specified `cond <breakpoint> -hitcount <operator> <number>` (or by setting the HitCond field when amending a breakpoint via the API), is a constraint on the number of times the breakpoint has been hit.
When a breakpoint location is encountered during the execution of the program, the debugger will:
* Evaluate the normal condition
* Stop if there is an error while evaluating the normal condition
* If the normal condition evaluates to true the hit count is incremented
* Evaluate the hitcount condition
* If the hitcount condition is also satisfied stop the execution at the breakpoint

View File

@ -119,6 +119,7 @@ Delve defines two special variables:
* `runtime.curg` evaluates to the 'g' struct for the current goroutine, in particular `runtime.curg.goid` is the goroutine id of the current goroutine.
* `runtime.frameoff` is the offset of the frame's base address from the bottom of the stack.
* `delve.bphitcount[X]` is the total hitcount for breakpoint X, which can be either an ID or the breakpoint name as a string.
## Access to variables from previous frames

View File

@ -340,3 +340,25 @@ var = eval(
{"FollowPointers":True, "MaxVariableRecurse":2, "MaxStringLen":100, "MaxArrayValues":10, "MaxStructFields":100}
)
```
## Chain breakpoints
Chain a number of breakpoints such that breakpoint n+1 is only hit after breakpoint n is hit:
```python
def command_breakchain(*args):
v = args.split(" ")
bp = get_breakpoint(int(v[0]), "").Breakpoint
bp.HitCond = "== 1"
amend_breakpoint(bp)
for i in range(1, len(v)):
bp = get_breakpoint(int(v[i]), "").Breakpoint
if i != len(v)-1:
bp.HitCond = "== 1"
bp.Cond = "delve.bphitcount[" + v[i-1] + "] > 0"
amend_breakpoint(bp)
```
To be used as `chain 1 2 3` where `1`, `2`, and `3` are IDs of breakpoints to chain together.

View File

@ -0,0 +1,31 @@
package main
import "fmt"
func breakfunc1() {
fmt.Println("breakfunc1")
}
func breakfunc2() {
fmt.Println("breakfunc2")
}
func breakfunc3() {
fmt.Println("breakfunc3")
}
func main() {
breakfunc2()
breakfunc3()
breakfunc1() // hit
breakfunc3()
breakfunc1()
breakfunc2() // hit
breakfunc1()
breakfunc3() // hit
breakfunc1()
breakfunc2()
}

View File

@ -0,0 +1,13 @@
def command_chain(args):
v = args.split(" ")
bp = get_breakpoint(int(v[0]), "").Breakpoint
bp.HitCond = "== 1"
amend_breakpoint(bp)
for i in range(1, len(v)):
bp = get_breakpoint(int(v[i]), "").Breakpoint
if i != len(v)-1:
bp.HitCond = "== 1"
bp.Cond = "delve.bphitcount[" + v[i-1] + "] > 0"
amend_breakpoint(bp)

View File

@ -11,12 +11,14 @@ import (
"go/printer"
"go/token"
"reflect"
"strconv"
"github.com/go-delve/delve/pkg/astutil"
"github.com/go-delve/delve/pkg/dwarf/godwarf"
"github.com/go-delve/delve/pkg/dwarf/op"
"github.com/go-delve/delve/pkg/dwarf/reader"
"github.com/go-delve/delve/pkg/goversion"
"github.com/go-delve/delve/pkg/proc/evalop"
"github.com/go-delve/delve/pkg/proc/internal/ebpf"
)
@ -938,6 +940,24 @@ func (bpmap *BreakpointMap) HasHWBreakpoints() bool {
return false
}
func totalHitCountByName(lbpmap map[int]*LogicalBreakpoint, s string) (uint64, error) {
for _, bp := range lbpmap {
if bp.Name == s {
return bp.TotalHitCount, nil
}
}
return 0, fmt.Errorf("could not find breakpoint named %q", s)
}
func totalHitCountByID(lbpmap map[int]*LogicalBreakpoint, id int) (uint64, error) {
for _, bp := range lbpmap {
if bp.LogicalID == int(id) {
return bp.TotalHitCount, nil
}
}
return 0, fmt.Errorf("could not find breakpoint with ID = %d", id)
}
// BreakpointState describes the state of a breakpoint in a thread.
type BreakpointState struct {
*Breakpoint
@ -1081,6 +1101,8 @@ type LogicalBreakpoint struct {
// condSatisfiable is true when 'cond && hitCond' can potentially be true.
condSatisfiable bool
// condUsesHitCounts is true when 'cond' uses breakpoint hitcounts
condUsesHitCounts bool
UserData interface{} // Any additional information about the breakpoint
// Name of root function from where tracing needs to be done
@ -1123,15 +1145,135 @@ func (lbp *LogicalBreakpoint) Cond() string {
return buf.String()
}
func breakpointConditionSatisfiable(lbp *LogicalBreakpoint) bool {
if lbp.hitCond == nil || lbp.HitCondPerG {
func breakpointConditionSatisfiable(lbpmap map[int]*LogicalBreakpoint, lbp *LogicalBreakpoint) bool {
if lbp.hitCond != nil && !lbp.HitCondPerG {
switch lbp.hitCond.Op {
case token.EQL, token.LEQ:
if int(lbp.TotalHitCount) >= lbp.hitCond.Val {
return false
}
case token.LSS:
if int(lbp.TotalHitCount) >= lbp.hitCond.Val-1 {
return false
}
}
}
if !lbp.condUsesHitCounts {
return true
}
switch lbp.hitCond.Op {
case token.EQL, token.LEQ:
return int(lbp.TotalHitCount) < lbp.hitCond.Val
case token.LSS:
return int(lbp.TotalHitCount) < lbp.hitCond.Val-1
toint := func(x ast.Expr) (uint64, bool) {
lit, ok := x.(*ast.BasicLit)
if !ok || lit.Kind != token.INT {
return 0, false
}
n, err := strconv.Atoi(lit.Value)
return uint64(n), err == nil && n >= 0
}
return true
hitcountexpr := func(x ast.Expr) (uint64, bool) {
idx, ok := x.(*ast.IndexExpr)
if !ok {
return 0, false
}
selx, ok := idx.X.(*ast.SelectorExpr)
if !ok {
return 0, false
}
ident, ok := selx.X.(*ast.Ident)
if !ok || ident.Name != evalop.BreakpointHitCountVarNamePackage || selx.Sel.Name != evalop.BreakpointHitCountVarName {
return 0, false
}
lit, ok := idx.Index.(*ast.BasicLit)
if !ok {
return 0, false
}
switch lit.Kind {
case token.INT:
n, _ := strconv.Atoi(lit.Value)
thc, err := totalHitCountByID(lbpmap, n)
return thc, err == nil
case token.STRING:
v, _ := strconv.Unquote(lit.Value)
thc, err := totalHitCountByName(lbpmap, v)
return thc, err == nil
default:
return 0, false
}
}
var satisf func(n ast.Node) bool
satisf = func(n ast.Node) bool {
parexpr, ok := n.(*ast.ParenExpr)
if ok {
return satisf(parexpr.X)
}
binexpr, ok := n.(*ast.BinaryExpr)
if !ok {
return true
}
switch binexpr.Op {
case token.AND:
return satisf(binexpr.X) && satisf(binexpr.Y)
case token.OR:
if !satisf(binexpr.X) {
return false
}
if !satisf(binexpr.Y) {
return false
}
return true
case token.EQL, token.LEQ, token.LSS, token.NEQ, token.GTR, token.GEQ:
default:
return true
}
hitcount, ok1 := hitcountexpr(binexpr.X)
val, ok2 := toint(binexpr.Y)
if !ok1 || !ok2 {
return true
}
switch binexpr.Op {
case token.EQL:
return hitcount == val
case token.LEQ:
return hitcount <= val
case token.LSS:
return hitcount < val
case token.NEQ:
return hitcount != val
case token.GTR:
return hitcount > val
case token.GEQ:
return hitcount >= val
}
return true
}
return satisf(lbp.cond)
}
func breakpointConditionUsesHitCounts(lbp *LogicalBreakpoint) bool {
if lbp.cond == nil {
return false
}
r := false
ast.Inspect(lbp.cond, func(n ast.Node) bool {
if r {
return false
}
seln, ok := n.(*ast.SelectorExpr)
if ok {
ident, ok := seln.X.(*ast.Ident)
if ok {
if ident.Name == evalop.BreakpointHitCountVarNamePackage && seln.Sel.Name == evalop.BreakpointHitCountVarName {
r = true
return false
}
}
}
return true
})
return r
}

View File

@ -1254,6 +1254,9 @@ func (stack *evalStack) executeOp() {
case *evalop.PushDebugPinner:
stack.push(stack.debugPinner)
case *evalop.PushBreakpointHitCount:
stack.push(newVariable(evalop.BreakpointHitCountVarNameQualified, fakeAddressUnresolv, godwarf.FakeSliceType(godwarf.FakeBasicType("uint", 64)), scope.BinInfo, scope.Mem))
default:
stack.err = fmt.Errorf("internal debugger error: unknown eval opcode: %#v", op)
}
@ -2072,6 +2075,33 @@ func (scope *EvalScope) evalIndex(op *evalop.Index, stack *evalStack) {
return
}
if xev.Name == evalop.BreakpointHitCountVarNameQualified {
if idxev.Kind == reflect.String {
s := constant.StringVal(idxev.Value)
thc, err := totalHitCountByName(scope.target.Breakpoints().Logical, s)
if err == nil {
stack.push(newConstant(constant.MakeUint64(thc), scope.Mem))
}
stack.err = err
return
}
n, err := idxev.asInt()
if err != nil {
n2, err := idxev.asUint()
if err != nil {
stack.err = fmt.Errorf("can not index %s with %s", xev.Name, astutil.ExprToString(op.Node.Index))
return
}
n = int64(n2)
}
thc, err := totalHitCountByID(scope.target.Breakpoints().Logical, int(n))
if err == nil {
stack.push(newConstant(constant.MakeUint64(thc), scope.Mem))
}
stack.err = err
return
}
if xev.Flags&VariableCPtr == 0 {
xev = xev.maybeDereference()
}

View File

@ -17,8 +17,14 @@ import (
)
var (
ErrFuncCallNotAllowed = errors.New("function calls not allowed without using 'call'")
DebugPinnerFunctionName = "runtime.debugPinnerV1"
ErrFuncCallNotAllowed = errors.New("function calls not allowed without using 'call'")
)
const (
BreakpointHitCountVarNamePackage = "delve"
BreakpointHitCountVarName = "bphitcount"
BreakpointHitCountVarNameQualified = BreakpointHitCountVarNamePackage + "." + BreakpointHitCountVarName
DebugPinnerFunctionName = "runtime.debugPinnerV1"
)
type compileCtx struct {
@ -308,6 +314,9 @@ func (ctx *compileCtx) compileAST(t ast.Expr, toplevel bool) error {
case x.Name == "runtime" && node.Sel.Name == "rangeParentOffset":
ctx.pushOp(&PushRangeParentOffset{})
case x.Name == BreakpointHitCountVarNamePackage && node.Sel.Name == BreakpointHitCountVarName:
ctx.pushOp(&PushBreakpointHitCount{})
default:
ctx.pushOp(&PushPackageVarOrSelect{Name: x.Name, Sel: node.Sel.Name})
}

View File

@ -326,3 +326,10 @@ type PushPinAddress struct {
}
func (*PushPinAddress) depthCheck() (npop, npush int) { return 0, 1 }
// PushBreakpointHitCount pushes a special array containing the hit counts
// of breakpoints.
type PushBreakpointHitCount struct {
}
func (*PushBreakpointHitCount) depthCheck() (npop, npush int) { return 0, 1 }

View File

@ -98,6 +98,17 @@ func withTestProcessArgs(name string, t testing.TB, wd string, args []string, bu
buildFlags |= protest.BuildModePIE
}
fixture := protest.BuildFixture(name, buildFlags)
grp := startTestProcessArgs(fixture, t, wd, args)
defer func() {
grp.Detach(true)
}()
fn(grp.Selected, grp, fixture)
}
func startTestProcessArgs(fixture protest.Fixture, t testing.TB, wd string, args []string) *proc.TargetGroup {
var grp *proc.TargetGroup
var err error
var tracedir string
@ -118,12 +129,7 @@ func withTestProcessArgs(name string, t testing.TB, wd string, args []string, bu
if err != nil {
t.Fatal("Launch():", err)
}
defer func() {
grp.Detach(true)
}()
fn(grp.Selected, grp, fixture)
return grp
}
func getRegisters(p *proc.Target, t *testing.T) proc.Registers {
@ -250,6 +256,7 @@ func setFunctionBreakpoint(p *proc.Target, t testing.TB, fname string) *proc.Bre
if err != nil {
t.Fatalf("FindFunctionLocation(%s): %v", fname, err)
}
bp.Logical.Set.FunctionName = fname
return bp
}
@ -5664,3 +5671,98 @@ func TestStackwatchClearBug(t *testing.T) {
}
})
}
func TestChainedBreakpoint(t *testing.T) {
assertCallerLine := func(t *testing.T, p *proc.Target, pos string, tgt int) {
t.Helper()
frames, err := proc.ThreadStacktrace(p, p.CurrentThread(), 5)
assertNoError(err, t, "ThreadStacktrace")
t.Logf("%s: %s:%d", pos, frames[1].Call.File, frames[1].Call.Line)
if frames[1].Call.Line != tgt {
t.Fatalf("wrong line number, expected %d", tgt)
}
}
withTestProcess("bphitcountchain", t, func(p *proc.Target, grp *proc.TargetGroup, fixture protest.Fixture) {
numphys := func(lbp *proc.LogicalBreakpoint) int {
count := 0
for _, bp := range p.Breakpoints().M {
if bp.LogicalID() == lbp.LogicalID {
count++
}
}
return count
}
bp := setFunctionBreakpoint(p, t, "main.breakfunc3")
lbp3 := bp.Logical
bp = setFunctionBreakpoint(p, t, "main.breakfunc2")
lbp2 := bp.Logical
bp = setFunctionBreakpoint(p, t, "main.breakfunc1")
lbp1 := bp.Logical
assertPhysCount := func(lbp1cnt, lbp2cnt, lbp3cnt int) {
t.Helper()
t.Logf("lbp1: %d lbp2: %d lbp3: %d", numphys(lbp1), numphys(lbp2), numphys(lbp3))
if numphys(lbp1) != lbp1cnt || numphys(lbp2) != lbp2cnt || numphys(lbp3) != lbp3cnt {
t.Fatal("wrong number of physical breakpoints")
}
}
assertNoError(grp.ChangeBreakpointCondition(lbp1, "", "== 1", false), t, "ChangeBreakpointCondition")
assertNoError(grp.ChangeBreakpointCondition(lbp2, fmt.Sprintf("delve.bphitcount[%d] > 0", lbp1.LogicalID), "== 1", false), t, "ChangeBreakpointCondition")
assertNoError(grp.ChangeBreakpointCondition(lbp3, fmt.Sprintf("delve.bphitcount[%d] > 0", lbp2.LogicalID), "== 1", false), t, "ChangeBreakpointCondition")
assertPhysCount(1, 0, 0)
assertNoError(grp.Continue(), t, "Continue 1")
assertCallerLine(t, p, "continue 1", 21)
assertNoError(grp.Continue(), t, "Continue 2")
assertCallerLine(t, p, "continue 2", 25)
assertPhysCount(0, 1, 0)
assertNoError(grp.Continue(), t, "Continue 3")
assertCallerLine(t, p, "continue 3", 28)
assertPhysCount(0, 0, 1)
err := grp.Continue()
if !errors.As(err, &proc.ErrProcessExited{}) {
assertNoError(err, t, "Continue 4")
}
// === Restart ===
t.Logf("=== Restart ===")
grp2 := startTestProcessArgs(fixture, t, ".", []string{})
proc.Restart(grp2, grp, func(lbp *proc.LogicalBreakpoint, err error) {
t.Fatalf("discarded logical breakpoint %v: %v", lbp, err)
})
grp = grp2
p = grp.Selected
assertPhysCount(1, 0, 0)
assertNoError(grp.Continue(), t, "Continue 1")
assertCallerLine(t, p, "continue 1", 21)
assertNoError(grp.Continue(), t, "Continue 2")
assertCallerLine(t, p, "continue 2", 25)
assertPhysCount(0, 1, 0)
assertNoError(grp.Continue(), t, "Continue 3")
assertCallerLine(t, p, "continue 3", 28)
assertPhysCount(0, 0, 1)
err = grp.Continue()
if !errors.As(err, &proc.ErrProcessExited{}) {
assertNoError(err, t, "Continue 4")
}
})
}

View File

@ -71,6 +71,7 @@ func NewGroup(procgrp ProcessGroup, cfg NewTargetGroupConfig) (*TargetGroup, Add
// Breakpoints that can not be set will be discarded, if discard is not nil
// it will be called for each discarded breakpoint.
func Restart(grp, oldgrp *TargetGroup, discard func(*LogicalBreakpoint, error)) {
toenable := []*LogicalBreakpoint{}
for _, bp := range oldgrp.LogicalBreakpoints {
if _, ok := grp.LogicalBreakpoints[bp.LogicalID]; ok {
continue
@ -80,14 +81,17 @@ func Restart(grp, oldgrp *TargetGroup, discard func(*LogicalBreakpoint, error))
bp.HitCount = make(map[int64]uint64)
bp.Set.PidAddrs = nil // breakpoints set through a list of addresses can not be restored after a restart
if bp.enabled {
bp.condSatisfiable = breakpointConditionSatisfiable(bp)
err := grp.enableBreakpoint(bp)
if err != nil {
if discard != nil {
discard(bp, err)
}
delete(grp.LogicalBreakpoints, bp.LogicalID)
toenable = append(toenable, bp)
}
}
for _, bp := range toenable {
bp.condSatisfiable = breakpointConditionSatisfiable(grp.LogicalBreakpoints, bp)
err := grp.enableBreakpoint(bp)
if err != nil {
if discard != nil {
discard(bp, err)
}
delete(grp.LogicalBreakpoints, bp.LogicalID)
}
}
if oldgrp.followExecEnabled {
@ -265,7 +269,7 @@ func (grp *TargetGroup) SetBreakpointEnabled(lbp *LogicalBreakpoint, enabled boo
err = grp.disableBreakpoint(lbp)
case !lbp.enabled && enabled:
lbp.enabled = true
lbp.condSatisfiable = breakpointConditionSatisfiable(lbp)
lbp.condSatisfiable = breakpointConditionSatisfiable(grp.LogicalBreakpoints, lbp)
err = grp.enableBreakpoint(lbp)
}
return
@ -424,12 +428,14 @@ func (grp *TargetGroup) ChangeBreakpointCondition(lbp *LogicalBreakpoint, cond,
lbp.HitCondPerG = hitCondPerG
}
lbp.condUsesHitCounts = breakpointConditionUsesHitCounts(lbp)
if lbp.enabled {
switch {
case lbp.condSatisfiable && !breakpointConditionSatisfiable(lbp):
case lbp.condSatisfiable && !breakpointConditionSatisfiable(grp.LogicalBreakpoints, lbp):
lbp.condSatisfiable = false
grp.disableBreakpoint(lbp)
case !lbp.condSatisfiable && breakpointConditionSatisfiable(lbp):
case !lbp.condSatisfiable && breakpointConditionSatisfiable(grp.LogicalBreakpoints, lbp):
lbp.condSatisfiable = true
grp.enableBreakpoint(lbp)
}
@ -483,12 +489,18 @@ func parseHitCondition(hitCond string) (token.Token, int, error) {
func (grp *TargetGroup) manageUnsatisfiableBreakpoints() error {
for _, lbp := range grp.LogicalBreakpoints {
if lbp.enabled {
if lbp.condSatisfiable && !breakpointConditionSatisfiable(lbp) {
if lbp.condSatisfiable && !breakpointConditionSatisfiable(grp.LogicalBreakpoints, lbp) {
lbp.condSatisfiable = false
err := grp.disableBreakpoint(lbp)
if err != nil {
return err
}
} else if lbp.condUsesHitCounts && !lbp.condSatisfiable && breakpointConditionSatisfiable(grp.LogicalBreakpoints, lbp) {
lbp.condSatisfiable = true
err := grp.enableBreakpoint(lbp)
if err != nil {
return err
}
}
}
}

View File

@ -1648,6 +1648,10 @@ func (v *Variable) loadSliceInfo(t *godwarf.SliceType) {
}
}
if v.Addr == fakeAddressUnresolv && v.fieldType == nil {
return
}
v.stride = v.fieldType.Size()
if t, ok := v.fieldType.(*godwarf.PtrType); ok {
v.stride = t.ByteSize

View File

@ -508,7 +508,7 @@ The command 'on x -edit' can be used to edit the list of commands executed when
Specifies that the breakpoint, tracepoint or watchpoint should break only if the boolean expression is true.
See Documentation/cli/expr.md for a description of supported expressions.
See Documentation/cli/expr.md for a description of supported expressions and Documentation/cli/cond.md for a description of how breakpoint conditions are evaluated.
With the -hitcount option a condition on the breakpoint hit count can be set, the following operators are supported

View File

@ -139,6 +139,7 @@ func testStarlarkAmendBreakpoint(t *testing.T, term *FakeTerminal) {
if !strings.Contains(out, "Stacktrace:2") || !strings.Contains(out, `HitCond:"== 2"`) {
t.Fatalf("wrong output")
}
term.MustExec("clear afuncbreak")
}
func TestStarlarkVariable(t *testing.T) {
@ -256,3 +257,46 @@ v.Children[0].Children[0].Value.XXX
}
})
}
func TestStarlarkChainBreakpointsExample(t *testing.T) {
withTestTerminal("bphitcountchain", t, func(term *FakeTerminal) {
term.MustExec("source " + findStarFile("chain_breakpoints"))
term.MustExec("break main.breakfunc1")
term.MustExec("break main.breakfunc2")
term.MustExec("break main.breakfunc3")
term.MustExec("chain 1 2 3")
out := term.MustExec("breakpoints")
t.Log(out)
numphys := func(id int) int {
bp, err := term.client.GetBreakpoint(id)
if err != nil {
t.Fatalf("Error getting breakpoint %d: %v", id, err)
}
return len(bp.Addrs)
}
assertPhysCount := func(lbp1cnt, lbp2cnt, lbp3cnt int) {
t.Helper()
t.Logf("lbp1: %d lbp2: %d lbp3: %d", numphys(1), numphys(2), numphys(3))
if numphys(1) != lbp1cnt || numphys(2) != lbp2cnt || numphys(3) != lbp3cnt {
t.Fatal("Wrong number of physical breakpoints")
}
}
assertPhysCount(1, 0, 0)
term.MustExec("continue")
listIsAt(t, term, "frame 1 list", 21, -1, -1)
term.MustExec("continue")
listIsAt(t, term, "frame 1 list", 25, -1, -1)
assertPhysCount(0, 1, 0)
term.MustExec("continue")
listIsAt(t, term, "frame 1 list", 28, -1, -1)
assertPhysCount(0, 0, 1)
})
}