proc,service,terminal: add events call use it for dld notifications (#3980)

Add a GetEvents method that can be used to retrieve debug events, adds
events for downloading debug info through debuginfod and shows those
events in the command line client as well as console messages in DAP.

In the future this events mechanism can be used to unify EBPF
tracepoints with old style tracepoints.

Update #3906

Documentation/cli/starlark.md

@@ -25,7 +25,7 @@ cancel_next() | Equivalent to API call [CancelNext](https://pkg.go.dev/github.co
 checkpoint(Where) | Equivalent to API call [Checkpoint](https://pkg.go.dev/github.com/go-delve/delve/service/rpc2#RPCServer.Checkpoint)
 clear_breakpoint(Id, Name) | Equivalent to API call [ClearBreakpoint](https://pkg.go.dev/github.com/go-delve/delve/service/rpc2#RPCServer.ClearBreakpoint)
 clear_checkpoint(ID) | Equivalent to API call [ClearCheckpoint](https://pkg.go.dev/github.com/go-delve/delve/service/rpc2#RPCServer.ClearCheckpoint)
-raw_command(Name, ThreadID, GoroutineID, ReturnInfoLoadConfig, Expr, UnsafeCall) | Equivalent to API call [Command](https://pkg.go.dev/github.com/go-delve/delve/service/rpc2#RPCServer.Command)
+raw_command(Name, ThreadID, GoroutineID, ReturnInfoLoadConfig, Expr, WithEvents, UnsafeCall) | Equivalent to API call [Command](https://pkg.go.dev/github.com/go-delve/delve/service/rpc2#RPCServer.Command)
 create_breakpoint(Breakpoint, LocExpr, SubstitutePathRules, Suspended) | Equivalent to API call [CreateBreakpoint](https://pkg.go.dev/github.com/go-delve/delve/service/rpc2#RPCServer.CreateBreakpoint)
 create_ebpf_tracepoint(FunctionName) | Equivalent to API call [CreateEBPFTracepoint](https://pkg.go.dev/github.com/go-delve/delve/service/rpc2#RPCServer.CreateEBPFTracepoint)
 create_watchpoint(Scope, Expr, Type) | Equivalent to API call [CreateWatchpoint](https://pkg.go.dev/github.com/go-delve/delve/service/rpc2#RPCServer.CreateWatchpoint)

cmd/dlv/dlv_test.go

@@ -454,7 +454,7 @@ func findCallCall(fndecl *ast.FuncDecl) *ast.CallExpr {
 			continue
 		}
 		fun, issel := callx.Fun.(*ast.SelectorExpr)
-		if !issel || fun.Sel.Name != "call" {
+		if !issel || (fun.Sel.Name != "call" && fun.Sel.Name != "callWhileDrainingEvents") {
 			continue
 		}
 		return callx
@@ -467,9 +467,6 @@ func qf(*types.Package) string {
 }
 
 func TestTypecheckRPC(t *testing.T) {
-	if goversion.VersionAfterOrEqual(runtime.Version(), 1, 24) {
-		t.Skip("disabled due to export format changes")
-	}
 	fset := &token.FileSet{}
 	cfg := &packages.Config{
 		Mode: packages.NeedSyntax | packages.NeedTypesInfo | packages.NeedName | packages.NeedCompiledGoFiles | packages.NeedTypes,
@@ -511,7 +508,7 @@ func TestTypecheckRPC(t *testing.T) {
 		case "Continue", "Rewind":
 			// wrappers over continueDir
 			continue
-		case "SetReturnValuesLoadConfig", "Disconnect":
+		case "SetReturnValuesLoadConfig", "Disconnect", "SetEventsFn":
 			// support functions
 			continue
 		}

pkg/proc/bininfo.go

@@ -115,6 +115,7 @@ type BinaryInfo struct {
 
 	debugPinnerFn *Function
 	logger        logflags.Logger
+	eventsFn      func(*Event)
 }
 
 var (
@@ -1559,7 +1560,19 @@ func (bi *BinaryInfo) openSeparateDebugInfo(image *Image, exe *elf.File, debugIn
 	// has debuginfod so that we can use that in order to find any relevant debug information.
 	if debugFilePath == "" {
 		var err error
-		debugFilePath, err = debuginfod.GetDebuginfo(image.BuildID)
+		var notify func(string)
+		if bi.eventsFn != nil {
+			notify = func(s string) {
+				bi.eventsFn(&Event{
+					Kind: EventBinaryInfoDownload,
+					BinaryInfoDownloadEventDetails: &BinaryInfoDownloadEventDetails{
+						ImagePath: image.Path,
+						Progress:  s,
+					},
+				})
+			}
+		}
+		debugFilePath, err = debuginfod.GetDebuginfo(notify, image.BuildID)
 		if err != nil {
 			return nil, nil, ErrNoDebugInfoFound
 		}

pkg/proc/debuginfod/debuginfod.go

@@ -1,17 +1,51 @@
 package debuginfod
 
 import (
+	"bufio"
+	"bytes"
+	"os"
 	"os/exec"
 	"strings"
+	"time"
 )
 
 const debuginfodFind = "debuginfod-find"
+const notificationThrottle time.Duration = 1 * time.Second
 
-func execFind(args ...string) (string, error) {
+func execFind(notify func(string), args ...string) (string, error) {
 	if _, err := exec.LookPath(debuginfodFind); err != nil {
 		return "", err
 	}
 	cmd := exec.Command(debuginfodFind, args...)
+	if notify != nil {
+		cmd.Env = append(os.Environ(), "DEBUGINFOD_PROGRESS=yes")
+		stderr, err := cmd.StderrPipe()
+		if err != nil {
+			return "", err
+		}
+		s := bufio.NewScanner(stderr)
+		s.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
+			if atEOF && len(data) == 0 {
+				return 0, nil, nil
+			}
+			if i := bytes.IndexAny(data, "\n\r"); i >= 0 {
+				return i + 1, dropCR(data[0:i]), nil
+			}
+			if atEOF {
+				return len(data), dropCR(data), nil
+			}
+			return 0, nil, nil
+		})
+		go func() {
+			var tlast time.Time
+			for s.Scan() {
+				if time.Since(tlast) > notificationThrottle {
+					tlast = time.Now()
+					notify(string(s.Text()))
+				}
+			}
+		}()
+	}
 	out, err := cmd.Output() // ignore stderr
 	if err != nil {
 		return "", err
@@ -19,10 +53,15 @@ func execFind(args ...string) (string, error) {
 	return strings.TrimSpace(string(out)), err
 }
 
-func GetSource(buildid, filename string) (string, error) {
-	return execFind("source", buildid, filename)
+func dropCR(data []byte) []byte {
+	r, _ := bytes.CutSuffix(data, []byte{'\r'})
+	return r
 }
 
-func GetDebuginfo(buildid string) (string, error) {
-	return execFind("debuginfo", buildid)
+func GetSource(buildid, filename string) (string, error) {
+	return execFind(nil, "source", buildid, filename)
+}
+
+func GetDebuginfo(notify func(string), buildid string) (string, error) {
+	return execFind(notify, "debuginfo", buildid)
 }

pkg/proc/target_group.go

@@ -537,6 +537,12 @@ func (grp *TargetGroup) FollowExecEnabled() bool {
 	return grp.followExecEnabled
 }
 
+// SetEventsFn sets a function that is called to communicate events
+// happening while the target process is running.
+func (grp *TargetGroup) SetEventsFn(eventsFn func(*Event)) {
+	grp.Selected.BinInfo().eventsFn = eventsFn
+}
+
 // ValidTargets iterates through all valid targets in Group.
 type ValidTargets struct {
 	*Target
@@ -564,3 +570,22 @@ func (it *ValidTargets) Reset() {
 	it.Target = nil
 	it.start = 0
 }
+
+// Event is an event that happened during execution of the debugged program.
+type Event struct {
+	Kind EventKind
+	*BinaryInfoDownloadEventDetails
+}
+
+type EventKind uint8
+
+const (
+	EventResumed EventKind = iota
+	EventStopped
+	EventBinaryInfoDownload
+)
+
+// BinaryInfoDownloadEventDetails details of a BinaryInfoDownload event.
+type BinaryInfoDownloadEventDetails struct {
+	ImagePath, Progress string
+}

pkg/terminal/starbind/starlark_mapping.go

@@ -271,7 +271,13 @@ func (env *Env) starlarkPredeclare() (starlark.StringDict, map[string]string) {
 			}
 		}
 		if len(args) > 5 && args[5] != starlark.None {
-			err := unmarshalStarlarkValue(args[5], &rpcArgs.UnsafeCall, "UnsafeCall")
+			err := unmarshalStarlarkValue(args[5], &rpcArgs.WithEvents, "WithEvents")
+			if err != nil {
+				return starlark.None, decorateError(thread, err)
+			}
+		}
+		if len(args) > 6 && args[6] != starlark.None {
+			err := unmarshalStarlarkValue(args[6], &rpcArgs.UnsafeCall, "UnsafeCall")
 			if err != nil {
 				return starlark.None, decorateError(thread, err)
 			}
@@ -289,6 +295,8 @@ func (env *Env) starlarkPredeclare() (starlark.StringDict, map[string]string) {
 				err = unmarshalStarlarkValue(kv[1], &rpcArgs.ReturnInfoLoadConfig, "ReturnInfoLoadConfig")
 			case "Expr":
 				err = unmarshalStarlarkValue(kv[1], &rpcArgs.Expr, "Expr")
+			case "WithEvents":
+				err = unmarshalStarlarkValue(kv[1], &rpcArgs.WithEvents, "WithEvents")
 			case "UnsafeCall":
 				err = unmarshalStarlarkValue(kv[1], &rpcArgs.UnsafeCall, "UnsafeCall")
 			default:
@@ -304,7 +312,7 @@ func (env *Env) starlarkPredeclare() (starlark.StringDict, map[string]string) {
 		}
 		return env.interfaceToStarlarkValue(&rpcRet), nil
 	})
-	doc["raw_command"] = "builtin raw_command(Name, ThreadID, GoroutineID, ReturnInfoLoadConfig, Expr, UnsafeCall)\n\nraw_command interrupts, continues and steps through the program."
+	doc["raw_command"] = "builtin raw_command(Name, ThreadID, GoroutineID, ReturnInfoLoadConfig, Expr, WithEvents, UnsafeCall)\n\nraw_command interrupts, continues and steps through the program."
 	r["create_breakpoint"] = starlark.NewBuiltin("create_breakpoint", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
 		if err := isCancelled(thread); err != nil {
 			return starlark.None, decorateError(thread, err)

pkg/terminal/terminal.go

@@ -122,6 +122,23 @@ func New(client service.Client, conf *config.Config) *Term {
 		if state, err := client.GetState(); err == nil {
 			t.oldPid = state.Pid
 		}
+		firstEventBinaryInfoDownload := true
+		client.SetEventsFn(func(event *api.Event) {
+			switch event.Kind {
+			case api.EventResumed:
+				firstEventBinaryInfoDownload = true
+			case api.EventBinaryInfoDownload:
+				if !firstEventBinaryInfoDownload {
+					fmt.Fprintf(t.stdout, "\r")
+				}
+				fmt.Fprintf(t.stdout, "Downloading debug info for %s: %s", event.BinaryInfoDownloadEventDetails.ImagePath, event.BinaryInfoDownloadEventDetails.Progress)
+				firstEventBinaryInfoDownload = false
+			case api.EventStopped:
+				if !firstEventBinaryInfoDownload {
+					fmt.Fprintf(t.stdout, "\n")
+				}
+			}
+		})
 	}
 
 	t.starlarkEnv = starbind.New(starlarkContext{t}, t.stdout)

service/api/conversions.go

@@ -462,3 +462,16 @@ func ConvertTarget(tgt *proc.Target, convertThreadBreakpoint func(proc.Thread) *
 		CurrentThread: ConvertThread(tgt.CurrentThread(), convertThreadBreakpoint(tgt.CurrentThread())),
 	}
 }
+
+func ConvertEvent(event *proc.Event) *Event {
+	r := &Event{Kind: EventKind(event.Kind)}
+
+	if event.BinaryInfoDownloadEventDetails != nil {
+		r.BinaryInfoDownloadEventDetails = &BinaryInfoDownloadEventDetails{
+			ImagePath: event.BinaryInfoDownloadEventDetails.ImagePath,
+			Progress:  event.BinaryInfoDownloadEventDetails.Progress,
+		}
+	}
+
+	return r
+}

service/api/types.go

@@ -411,6 +411,10 @@ type DebuggerCommand struct {
 	// Expr is the expression argument for a Call command
 	Expr string `json:"expr,omitempty"`
 
+	// If WithEvents is set events are generated that should be read by calling
+	// GetEvents.
+	WithEvents bool
+
 	// UnsafeCall disables parameter escape checking for function calls.
 	// Go objects can be allocated on the stack or on the heap. Heap objects
 	// can be used by any goroutine; stack objects can only be used by the
@@ -684,3 +688,22 @@ type GuessSubstitutePathIn struct {
 	ClientGOROOT            string
 	ClientModuleDirectories map[string]string
 }
+
+// Event is an event that happened during execution of the debugged program.
+type Event struct {
+	Kind EventKind
+	*BinaryInfoDownloadEventDetails
+}
+
+type EventKind uint8
+
+const (
+	EventResumed EventKind = iota
+	EventStopped
+	EventBinaryInfoDownload
+)
+
+// BinaryInfoDownloadEventDetails describes the details of a BinaryInfoDownloadEvent
+type BinaryInfoDownloadEventDetails struct {
+	ImagePath, Progress string
+}

service/client.go

@@ -166,6 +166,9 @@ type Client interface {
 	// SetReturnValuesLoadConfig sets the load configuration for return values.
 	SetReturnValuesLoadConfig(*api.LoadConfig)
 
+	// SetEventsFn sets a function that will be called whenever a debugger event is received.
+	SetEventsFn(func(*api.Event))
+
 	// IsMulticlient returns true if the headless instance is multiclient.
 	IsMulticlient() bool
 

service/dap/server.go

@@ -1398,7 +1398,7 @@ func (s *Session) halt() (*api.DebuggerState, error) {
 	s.config.log.Debug("halting")
 	// Only send a halt request if the debuggee is running.
 	if s.debugger.IsRunning() {
-		return s.debugger.Command(&api.DebuggerCommand{Name: api.Halt}, nil, nil)
+		return s.debugger.Command(&api.DebuggerCommand{Name: api.Halt}, nil, nil, nil)
 	}
 	s.config.log.Debug("process not running")
 	return s.debugger.State(false)
@@ -2102,7 +2102,7 @@ func (s *Session) stoppedOnBreakpointGoroutineID(state *api.DebuggerState) (int6
 // due to an error, so the server is ready to receive new requests.
 func (s *Session) stepUntilStopAndNotify(command string, threadId int, granularity dap.SteppingGranularity, allowNextStateChange *syncflag) {
 	defer allowNextStateChange.raise()
-	_, err := s.debugger.Command(&api.DebuggerCommand{Name: api.SwitchGoroutine, GoroutineID: int64(threadId)}, nil, s.conn.closedChan)
+	_, err := s.debugger.Command(&api.DebuggerCommand{Name: api.SwitchGoroutine, GoroutineID: int64(threadId)}, nil, s.conn.closedChan, nil)
 	if err != nil {
 		s.config.log.Errorf("Error switching goroutines while stepping: %v", err)
 		// If we encounter an error, we will have to send a stopped event
@@ -2972,7 +2972,8 @@ func (s *Session) doCall(goid, frame int, expr string) (*api.DebuggerState, []*p
 		Expr:                 expr,
 		UnsafeCall:           false,
 		GoroutineID:          int64(goid),
-	}, nil, s.conn.closedChan)
+		WithEvents:           true,
+	}, nil, s.conn.closedChan, s.convertDebuggerEvent)
 	if processExited(state, err) {
 		s.preTerminatedWG.Wait()
 		e := &dap.TerminatedEvent{Event: *newEvent("terminated")}
@@ -3775,7 +3776,7 @@ func (s *Session) resumeOnce(command string, allowNextStateChange *syncflag) (bo
 		state, err := s.debugger.State(false)
 		return false, state, err
 	}
-	state, err := s.debugger.Command(&api.DebuggerCommand{Name: command}, asyncSetupDone, s.conn.closedChan)
+	state, err := s.debugger.Command(&api.DebuggerCommand{Name: command, WithEvents: true}, asyncSetupDone, s.conn.closedChan, s.convertDebuggerEvent)
 	return true, state, err
 }
 
@@ -4046,6 +4047,19 @@ func (s *Session) toServerPath(path string) string {
 	return serverPath
 }
 
+func (s *Session) convertDebuggerEvent(event *proc.Event) {
+	switch event.Kind {
+	case proc.EventBinaryInfoDownload:
+		s.send(&dap.OutputEvent{
+			Event: *newEvent("output"),
+			Body: dap.OutputEventBody{
+				Output:   fmt.Sprintf("Download debug info for %s: %s\n", event.BinaryInfoDownloadEventDetails.ImagePath, event.BinaryInfoDownloadEventDetails.Progress),
+				Category: "console",
+			},
+		})
+	}
+}
+
 type logMessage struct {
 	format string
 	args   []string

service/dap/server_test.go

@@ -6819,7 +6819,7 @@ func launchDebuggerWithTargetRunning(t *testing.T, fixture string) (*protest.Fix
 	var err error
 	go func() {
 		t.Helper()
-		_, err = dbg.Command(&api.DebuggerCommand{Name: api.Continue}, running, nil)
+		_, err = dbg.Command(&api.DebuggerCommand{Name: api.Continue}, running, nil, nil)
 		select {
 		case <-running:
 		default:
@@ -7021,7 +7021,7 @@ func (s *MultiClientCloseServerMock) stop(t *testing.T) {
 	// they are part of dap.Session.
 	// We must take it down manually as if we are in rpccommon::ServerImpl::Stop.
 	if s.debugger.IsRunning() {
-		s.debugger.Command(&api.DebuggerCommand{Name: api.Halt}, nil, nil)
+		s.debugger.Command(&api.DebuggerCommand{Name: api.Halt}, nil, nil, nil)
 	}
 	s.debugger.Detach(true)
 }

service/debugger/debugger.go

@@ -1059,7 +1059,7 @@ func (d *Debugger) IsRunning() bool {
 }
 
 // Command handles commands which control the debugger lifecycle
-func (d *Debugger) Command(command *api.DebuggerCommand, resumeNotify chan struct{}, clientStatusCh chan struct{}) (state *api.DebuggerState, err error) {
+func (d *Debugger) Command(command *api.DebuggerCommand, resumeNotify chan struct{}, clientStatusCh chan struct{}, eventsFn func(*proc.Event)) (state *api.DebuggerState, err error) {
 	if command.Name == api.Halt {
 		// RequestManualStop does not invoke any ptrace syscalls, so it's safe to
 		// access the process directly.
@@ -1085,8 +1085,16 @@ func (d *Debugger) Command(command *api.DebuggerCommand, resumeNotify chan struc
 	d.setRunning(true)
 	defer d.setRunning(false)
 
+	d.target.SetEventsFn(nil)
 	if command.Name != api.SwitchGoroutine && command.Name != api.SwitchThread && command.Name != api.Halt {
 		d.target.ResumeNotify(resumeNotify)
+
+		if eventsFn != nil {
+			eventsFn(&proc.Event{Kind: proc.EventResumed})
+			defer eventsFn(&proc.Event{Kind: proc.EventStopped})
+		}
+
+		d.target.SetEventsFn(eventsFn)
 	} else if resumeNotify != nil {
 		close(resumeNotify)
 	}

service/rpc2/client.go

@@ -22,6 +22,7 @@ type RPCClient struct {
 	client *rpc.Client
 
 	retValLoadCfg *api.LoadConfig
+	eventsFn      func(*api.Event)
 }
 
 // Ensure the implementation satisfies the interface.
@@ -112,7 +113,7 @@ func (c *RPCClient) continueDir(cmd string) <-chan *api.DebuggerState {
 	go func() {
 		for {
 			out := new(CommandOut)
-			err := c.call("Command", &api.DebuggerCommand{Name: cmd, ReturnInfoLoadConfig: c.retValLoadCfg}, &out)
+			err := c.callWhileDrainingEvents("Command", &api.DebuggerCommand{Name: cmd, ReturnInfoLoadConfig: c.retValLoadCfg, WithEvents: c.eventsFn != nil}, &out)
 			state := out.State
 			if err != nil {
 				state.Err = err
@@ -146,45 +147,70 @@ func (c *RPCClient) continueDir(cmd string) <-chan *api.DebuggerState {
 	return ch
 }
 
+func (c *RPCClient) drainEvents() <-chan struct{} {
+	done := make(chan struct{})
+	if c.eventsFn == nil {
+		close(done)
+		return done
+	}
+	go func() {
+		defer close(done)
+		for {
+			out := new(GetEventsOut)
+			err := c.call("GetEvents", &GetEventsIn{}, &out)
+			if err != nil {
+				break
+			}
+			for _, event := range out.Events {
+				c.eventsFn(&event)
+				if event.Kind == api.EventStopped {
+					return
+				}
+			}
+		}
+	}()
+	return done
+}
+
 func (c *RPCClient) Next() (*api.DebuggerState, error) {
 	var out CommandOut
-	err := c.call("Command", api.DebuggerCommand{Name: api.Next, ReturnInfoLoadConfig: c.retValLoadCfg}, &out)
+	err := c.callWhileDrainingEvents("Command", api.DebuggerCommand{Name: api.Next, ReturnInfoLoadConfig: c.retValLoadCfg, WithEvents: c.eventsFn != nil}, &out)
 	return &out.State, err
 }
 
 func (c *RPCClient) ReverseNext() (*api.DebuggerState, error) {
 	var out CommandOut
-	err := c.call("Command", api.DebuggerCommand{Name: api.ReverseNext, ReturnInfoLoadConfig: c.retValLoadCfg}, &out)
+	err := c.callWhileDrainingEvents("Command", api.DebuggerCommand{Name: api.ReverseNext, ReturnInfoLoadConfig: c.retValLoadCfg, WithEvents: c.eventsFn != nil}, &out)
 	return &out.State, err
 }
 
 func (c *RPCClient) Step() (*api.DebuggerState, error) {
 	var out CommandOut
-	err := c.call("Command", api.DebuggerCommand{Name: api.Step, ReturnInfoLoadConfig: c.retValLoadCfg}, &out)
+	err := c.callWhileDrainingEvents("Command", api.DebuggerCommand{Name: api.Step, ReturnInfoLoadConfig: c.retValLoadCfg, WithEvents: c.eventsFn != nil}, &out)
 	return &out.State, err
 }
 
 func (c *RPCClient) ReverseStep() (*api.DebuggerState, error) {
 	var out CommandOut
-	err := c.call("Command", api.DebuggerCommand{Name: api.ReverseStep, ReturnInfoLoadConfig: c.retValLoadCfg}, &out)
+	err := c.callWhileDrainingEvents("Command", api.DebuggerCommand{Name: api.ReverseStep, ReturnInfoLoadConfig: c.retValLoadCfg, WithEvents: c.eventsFn != nil}, &out)
 	return &out.State, err
 }
 
 func (c *RPCClient) StepOut() (*api.DebuggerState, error) {
 	var out CommandOut
-	err := c.call("Command", api.DebuggerCommand{Name: api.StepOut, ReturnInfoLoadConfig: c.retValLoadCfg}, &out)
+	err := c.callWhileDrainingEvents("Command", api.DebuggerCommand{Name: api.StepOut, ReturnInfoLoadConfig: c.retValLoadCfg, WithEvents: c.eventsFn != nil}, &out)
 	return &out.State, err
 }
 
 func (c *RPCClient) ReverseStepOut() (*api.DebuggerState, error) {
 	var out CommandOut
-	err := c.call("Command", api.DebuggerCommand{Name: api.ReverseStepOut, ReturnInfoLoadConfig: c.retValLoadCfg}, &out)
+	err := c.callWhileDrainingEvents("Command", api.DebuggerCommand{Name: api.ReverseStepOut, ReturnInfoLoadConfig: c.retValLoadCfg, WithEvents: c.eventsFn != nil}, &out)
 	return &out.State, err
 }
 
 func (c *RPCClient) Call(goroutineID int64, expr string, unsafe bool) (*api.DebuggerState, error) {
 	var out CommandOut
-	err := c.call("Command", api.DebuggerCommand{Name: api.Call, ReturnInfoLoadConfig: c.retValLoadCfg, Expr: expr, UnsafeCall: unsafe, GoroutineID: goroutineID}, &out)
+	err := c.callWhileDrainingEvents("Command", api.DebuggerCommand{Name: api.Call, ReturnInfoLoadConfig: c.retValLoadCfg, Expr: expr, UnsafeCall: unsafe, GoroutineID: goroutineID, WithEvents: c.eventsFn != nil}, &out)
 	return &out.State, err
 }
 
@@ -194,7 +220,7 @@ func (c *RPCClient) StepInstruction(skipCalls bool) (*api.DebuggerState, error)
 	if skipCalls {
 		name = api.NextInstruction
 	}
-	err := c.call("Command", api.DebuggerCommand{Name: name}, &out)
+	err := c.callWhileDrainingEvents("Command", api.DebuggerCommand{Name: name, WithEvents: c.eventsFn != nil}, &out)
 	return &out.State, err
 }
 
@@ -204,7 +230,7 @@ func (c *RPCClient) ReverseStepInstruction(skipCalls bool) (*api.DebuggerState,
 	if skipCalls {
 		name = api.ReverseNextInstruction
 	}
-	err := c.call("Command", api.DebuggerCommand{Name: name}, &out)
+	err := c.callWhileDrainingEvents("Command", api.DebuggerCommand{Name: name, WithEvents: c.eventsFn != nil}, &out)
 	return &out.State, err
 }
 
@@ -492,6 +518,10 @@ func (c *RPCClient) SetReturnValuesLoadConfig(cfg *api.LoadConfig) {
 	c.retValLoadCfg = cfg
 }
 
+func (c *RPCClient) SetEventsFn(eventsFn func(*api.Event)) {
+	c.eventsFn = eventsFn
+}
+
 func (c *RPCClient) FunctionReturnLocations(fnName string) ([]uint64, error) {
 	var out FunctionReturnLocationsOut
 	err := c.call("FunctionReturnLocations", FunctionReturnLocationsIn{fnName}, &out)
@@ -667,6 +697,13 @@ func (c *RPCClient) call(method string, args, reply interface{}) error {
 	return c.client.Call("RPCServer."+method, args, reply)
 }
 
+func (c *RPCClient) callWhileDrainingEvents(method string, args, reply interface{}) error {
+	done := c.drainEvents()
+	err := c.call(method, args, reply)
+	<-done
+	return err
+}
+
 func (c *RPCClient) CallAPI(method string, args, reply interface{}) error {
 	return c.call(method, args, reply)
 }

service/rpc2/server.go

@@ -18,11 +18,14 @@ type RPCServer struct {
 	// config is all the information necessary to start the debugger and server.
 	config *service.Config
 	// debugger is a debugger service.
-	debugger *debugger.Debugger
+	debugger   *debugger.Debugger
+	eventsChan chan *proc.Event
 }
 
+const eventBufferSize = 100
+
 func NewServer(config *service.Config, debugger *debugger.Debugger) *RPCServer {
-	return &RPCServer{config, debugger}
+	return &RPCServer{config, debugger, make(chan *proc.Event, eventBufferSize)}
 }
 
 type ProcessPidIn struct {
@@ -127,7 +130,11 @@ type CommandOut struct {
 
 // Command interrupts, continues and steps through the program.
 func (s *RPCServer) Command(command api.DebuggerCommand, cb service.RPCCallback) {
-	st, err := s.debugger.Command(&command, cb.SetupDoneChan(), cb.DisconnectChan())
+	eventsFn := s.eventsFn
+	if !command.WithEvents {
+		eventsFn = nil
+	}
+	st, err := s.debugger.Command(&command, cb.SetupDoneChan(), cb.DisconnectChan(), eventsFn)
 	if err != nil {
 		cb.Return(nil, err)
 		return
@@ -137,6 +144,10 @@ func (s *RPCServer) Command(command api.DebuggerCommand, cb service.RPCCallback)
 	cb.Return(out, nil)
 }
 
+func (s *RPCServer) eventsFn(event *proc.Event) {
+	s.eventsChan <- event
+}
+
 type GetBufferedTracepointsIn struct {
 }
 
@@ -1160,3 +1171,26 @@ func (s *RPCServer) GuessSubstitutePath(arg GuessSubstitutePathIn, out *GuessSub
 	}
 	return nil
 }
+
+type GetEventsIn struct {
+}
+
+type GetEventsOut struct {
+	Events []api.Event
+}
+
+func (s *RPCServer) GetEvents(arg GetEventsIn, cb service.RPCCallback) {
+	close(cb.SetupDoneChan())
+	out := new(GetEventsOut)
+	out.Events = append(out.Events, *api.ConvertEvent(<-s.eventsChan))
+	for len(out.Events) < eventBufferSize {
+		select {
+		case event := <-s.eventsChan:
+			out.Events = append(out.Events, *api.ConvertEvent(event))
+		default:
+			cb.Return(out, nil)
+			return
+		}
+	}
+	cb.Return(out, nil)
+}

service/rpccommon/server.go

@@ -95,7 +95,7 @@ func (s *ServerImpl) Stop() error {
 		s.listener.Close()
 	}
 	if s.debugger.IsRunning() {
-		s.debugger.Command(&api.DebuggerCommand{Name: api.Halt}, nil, nil)
+		s.debugger.Command(&api.DebuggerCommand{Name: api.Halt}, nil, nil, nil)
 	}
 	kill := s.config.Debugger.AttachPid == 0
 	return s.debugger.Detach(kill)

service/rpccommon/suitablemethods.go

@@ -34,6 +34,7 @@ func suitableMethods2(s *rpc2.RPCServer, methods map[string]*methodType) {
 	methods["RPCServer.FunctionReturnLocations"] = &methodType{method: reflect.ValueOf(s.FunctionReturnLocations)}
 	methods["RPCServer.GetBreakpoint"] = &methodType{method: reflect.ValueOf(s.GetBreakpoint)}
 	methods["RPCServer.GetBufferedTracepoints"] = &methodType{method: reflect.ValueOf(s.GetBufferedTracepoints)}
+	methods["RPCServer.GetEvents"] = &methodType{method: reflect.ValueOf(s.GetEvents)}
 	methods["RPCServer.GetThread"] = &methodType{method: reflect.ValueOf(s.GetThread)}
 	methods["RPCServer.GuessSubstitutePath"] = &methodType{method: reflect.ValueOf(s.GuessSubstitutePath)}
 	methods["RPCServer.IsMulticlient"] = &methodType{method: reflect.ValueOf(s.IsMulticlient)}