From 6dcfd4c9799d9ad6aa0d283f5f7302f45cb42943 Mon Sep 17 00:00:00 2001 From: Utku Ozdemir Date: Wed, 19 Jun 2024 23:52:52 +0200 Subject: [PATCH] feat: handle all goroutine panics gracefully Convert goroutine panics to errors or error logs. Disallow usage of `golang.org/x/sync/errgroup` package in the backend by `depguard` linter. This linter configuration depends on: https://github.com/siderolabs/kres/pull/417 Rekres the project to include the feature (also bump Go to 1.22.4), but revert `PROTOBUF_GO_VERSION` and `GRPC_GATEWAY_VERSION` manually to not break the frontend. Disallowing the named `go` statement was not possible at the moment using existing linters, raised an issue in `forbidigo` for it: https://github.com/ashanbrown/forbidigo/issues/47 Closes siderolabs/omni#373. Signed-off-by: Utku Ozdemir --- .golangci.yml | 13 +- .kres.yaml | 15 +++ Dockerfile | 26 ++-- Makefile | 14 +-- client/.golangci.yml | 13 +- client/pkg/panichandler/panichandler.go | 86 +++++++++++++ client/pkg/panichandler/panichandler_test.go | 113 ++++++++++++++++++ cmd/omni/main.go | 9 +- go.mod | 2 +- internal/backend/grpc/management.go | 13 +- internal/backend/grpc/resource.go | 4 +- internal/backend/grpc/support.go | 4 +- .../backend/runtime/kubernetes/kubernetes.go | 4 +- .../omni/controllers/omni/etcd_backup.go | 4 +- .../omni/internal/etcdbackup/crypt/crypt.go | 5 +- .../internal/task/clustermachine/identity.go | 7 +- .../controllers/omni/internal/task/task.go | 6 +- .../controllers/omni/kubernetes_status.go | 13 +- .../kubernetes_upgrade_manifest_status.go | 5 +- .../backend/runtime/omni/external/state.go | 7 +- .../runtime/omni/external/state_test.go | 3 + .../backend/runtime/omni/pkg/check/etcd.go | 2 +- internal/backend/runtime/omni/state.go | 1 + internal/backend/runtime/omni/state_etcd.go | 5 +- .../runtime/omni/state_etcd_election.go | 9 +- .../backend/runtime/omni/virtual/computed.go | 8 +- .../virtual/pkg/producers/kubernetes_usage.go | 9 +- internal/backend/runtime/proxy_runtime.go | 10 +- internal/backend/server.go | 13 +- internal/pkg/errgroup/errgroup.go | 6 +- internal/pkg/grpcutil/server.go | 12 +- internal/pkg/logreceiver/logreceiver.go | 6 +- internal/pkg/siderolink/manager.go | 4 +- internal/pkg/siderolink/siderolink_test.go | 2 +- internal/version/data/tag | 2 +- 35 files changed, 361 insertions(+), 94 deletions(-) create mode 100644 client/pkg/panichandler/panichandler.go create mode 100644 client/pkg/panichandler/panichandler_test.go diff --git a/.golangci.yml b/.golangci.yml index 5a0e2f49..86277942 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2024-06-03T11:20:02Z by kres f292767. +# Generated on 2024-06-19T21:17:41Z by kres 4c9f215. # options for analysis running run: @@ -103,6 +103,17 @@ linters-settings: deny: - pkg: io/ioutil desc: "replaced by io and os packages since Go 1.16: https://tip.golang.org/doc/go1.16#ioutil" + prevent_sync_errgroup: + deny: + - desc: Use github.com/siderolabs/omni/client/pkg/panichandler.ErrGroup instead + pkg: golang.org/x/sync/errgroup + files: + - $all + - '!$test' + - '!**/integration-test/**' + - '!**/omnictl/**' + - '!**/panichandler/**' + list-mode: lax linters: enable-all: true diff --git a/.kres.yaml b/.kres.yaml index dd4d3183..d03f1bbc 100644 --- a/.kres.yaml +++ b/.kres.yaml @@ -362,6 +362,21 @@ spec: subdirectory: v1alpha1 genGateway: true --- +kind: golang.GolangciLint +spec: + depguardExtraRules: + prevent_sync_errgroup: + list-mode: lax + files: + - $all + - "!$test" + - "!**/integration-test/**" + - "!**/omnictl/**" + - "!**/panichandler/**" + deny: + - pkg: golang.org/x/sync/errgroup + desc: Use github.com/siderolabs/omni/client/pkg/panichandler.ErrGroup instead +--- kind: js.Build spec: licenseText: | diff --git a/Dockerfile b/Dockerfile index c5371ac1..fe06569a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ -# syntax = docker/dockerfile-upstream:1.7.1-labs +# syntax = docker/dockerfile-upstream:1.8.0-labs # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2024-06-06T12:40:29Z by kres 14c10c9-dirty. +# Generated on 2024-06-19T20:43:57Z by kres 4c9f215. ARG JS_TOOLCHAIN ARG TOOLCHAIN @@ -16,11 +16,11 @@ FROM --platform=${BUILDPLATFORM} ${JS_TOOLCHAIN} AS js-toolchain RUN apk --update --no-cache add bash curl protoc protobuf-dev go COPY ./go.mod . COPY ./go.sum . -ENV GOPATH /go -ENV PATH ${PATH}:/usr/local/go/bin +ENV GOPATH=/go +ENV PATH=${PATH}:/usr/local/go/bin # runs markdownlint -FROM docker.io/oven/bun:1.1.12-alpine AS lint-markdown +FROM docker.io/oven/bun:1.1.13-alpine AS lint-markdown WORKDIR /src RUN bun i markdownlint-cli@0.41.0 sentences-per-line@0.2.1 COPY .markdownlint.json . @@ -96,14 +96,14 @@ COPY ./frontend/postcss.config.js ./postcss.config.js # build tools FROM --platform=${BUILDPLATFORM} toolchain AS tools -ENV GO111MODULE on +ENV GO111MODULE=on ARG CGO_ENABLED -ENV CGO_ENABLED ${CGO_ENABLED} +ENV CGO_ENABLED=${CGO_ENABLED} ARG GOTOOLCHAIN -ENV GOTOOLCHAIN ${GOTOOLCHAIN} +ENV GOTOOLCHAIN=${GOTOOLCHAIN} ARG GOEXPERIMENT -ENV GOEXPERIMENT ${GOEXPERIMENT} -ENV GOPATH /go +ENV GOEXPERIMENT=${GOEXPERIMENT} +ENV GOPATH=/go ARG GOIMPORTS_VERSION RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/go/pkg go install golang.org/x/tools/cmd/goimports@v${GOIMPORTS_VERSION} RUN mv /go/bin/goimports /bin @@ -259,7 +259,7 @@ RUN FILES="$(gofumpt -l client)" && test -z "${FILES}" || (echo -e "Source code FROM base AS lint-golangci-lint WORKDIR /src COPY .golangci.yml . -ENV GOGC 50 +ENV GOGC=50 RUN golangci-lint config verify --config .golangci.yml RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.cache/golangci-lint --mount=type=cache,target=/go/pkg golangci-lint run --config .golangci.yml @@ -267,7 +267,7 @@ RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/r FROM base AS lint-golangci-lint-client WORKDIR /src/client COPY client/.golangci.yml . -ENV GOGC 50 +ENV GOGC=50 RUN golangci-lint config verify --config .golangci.yml RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.cache/golangci-lint --mount=type=cache,target=/go/pkg golangci-lint run --config .golangci.yml @@ -496,6 +496,6 @@ COPY --from=omni omni-linux-${TARGETARCH} /omni COPY --from=image-fhs / / COPY --from=image-ca-certificates / / COPY --from=omnictl-all / /omnictl/ -LABEL org.opencontainers.image.source https://github.com/siderolabs/omni +LABEL org.opencontainers.image.source=https://github.com/siderolabs/omni ENTRYPOINT ["/omni"] diff --git a/Makefile b/Makefile index f681676f..011b5365 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2024-06-06T10:06:36Z by kres dc18ad9-dirty. +# Generated on 2024-06-19T20:43:57Z by kres 4c9f215. # common variables @@ -20,15 +20,15 @@ REGISTRY_AND_USERNAME ?= $(REGISTRY)/$(USERNAME) PROTOBUF_GRPC_GATEWAY_TS_VERSION ?= 1.2.1 TESTPKGS ?= ./... JS_BUILD_ARGS ?= -PROTOBUF_GO_VERSION ?= 1.34.1 +PROTOBUF_GO_VERSION ?= 1.34.1 # reverted manually to 1.34.1 GRPC_GO_VERSION ?= 1.4.0 -GRPC_GATEWAY_VERSION ?= 2.19.1 +GRPC_GATEWAY_VERSION ?= 2.19.1 # reverted manually to 2.19.1 VTPROTOBUF_VERSION ?= 0.6.0 -GOIMPORTS_VERSION ?= 0.21.0 +GOIMPORTS_VERSION ?= 0.22.0 DEEPCOPY_VERSION ?= v0.5.6 -GOLANGCILINT_VERSION ?= v1.59.0 +GOLANGCILINT_VERSION ?= v1.59.1 GOFUMPT_VERSION ?= v0.6.0 -GO_VERSION ?= 1.22.3 +GO_VERSION ?= 1.22.4 GO_BUILDFLAGS ?= GO_LDFLAGS ?= CGO_ENABLED ?= 0 @@ -72,7 +72,7 @@ COMMON_ARGS += --build-arg=DEEPCOPY_VERSION="$(DEEPCOPY_VERSION)" COMMON_ARGS += --build-arg=GOLANGCILINT_VERSION="$(GOLANGCILINT_VERSION)" COMMON_ARGS += --build-arg=GOFUMPT_VERSION="$(GOFUMPT_VERSION)" COMMON_ARGS += --build-arg=TESTPKGS="$(TESTPKGS)" -JS_TOOLCHAIN ?= docker.io/oven/bun:1.1.12-alpine +JS_TOOLCHAIN ?= docker.io/oven/bun:1.1.13-alpine TOOLCHAIN ?= docker.io/golang:1.22-alpine # extra variables diff --git a/client/.golangci.yml b/client/.golangci.yml index 240b8294..86277942 100644 --- a/client/.golangci.yml +++ b/client/.golangci.yml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2024-06-03T12:34:17Z by kres f292767. +# Generated on 2024-06-19T21:17:41Z by kres 4c9f215. # options for analysis running run: @@ -103,6 +103,17 @@ linters-settings: deny: - pkg: io/ioutil desc: "replaced by io and os packages since Go 1.16: https://tip.golang.org/doc/go1.16#ioutil" + prevent_sync_errgroup: + deny: + - desc: Use github.com/siderolabs/omni/client/pkg/panichandler.ErrGroup instead + pkg: golang.org/x/sync/errgroup + files: + - $all + - '!$test' + - '!**/integration-test/**' + - '!**/omnictl/**' + - '!**/panichandler/**' + list-mode: lax linters: enable-all: true diff --git a/client/pkg/panichandler/panichandler.go b/client/pkg/panichandler/panichandler.go new file mode 100644 index 00000000..c885397c --- /dev/null +++ b/client/pkg/panichandler/panichandler.go @@ -0,0 +1,86 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package panichandler provides a panic handling errgroup. +package panichandler + +import ( + "context" + "errors" + "fmt" + "log" + "runtime/debug" + + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +const goroutinePanicked = "goroutine panicked" + +// ErrPanic is the error returned when a task panics. +var ErrPanic = errors.New(goroutinePanicked) + +// ErrGroup wraps golang.org/x/sync/errgroup.Group to handle panics by turning them into errors. +// It MUST be created using NewErrGroup. +type ErrGroup struct { + eg *errgroup.Group +} + +// NewErrGroup creates a new ErrGroup. +func NewErrGroup() *ErrGroup { + return &ErrGroup{eg: &errgroup.Group{}} +} + +// ErrGroupWithContext creates a new ErrGroup with the given context. +func ErrGroupWithContext(ctx context.Context) (*ErrGroup, context.Context) { + eg, ctx := errgroup.WithContext(ctx) + + return &ErrGroup{eg: eg}, ctx +} + +// Go runs the given function in a goroutine, handling panics by turning them into errors. +func (eg *ErrGroup) Go(f func() error) { + eg.eg.Go(func() (err error) { + defer func() { + if r := recover(); r != nil { + stack := debug.Stack() + + err = errors.Join(err, fmt.Errorf("%w: %s\n%s", ErrPanic, r, string(stack))) + } + }() + + return f() + }) +} + +// SetLimit sets the maximum number of goroutines that can run concurrently. +func (eg *ErrGroup) SetLimit(n int) { + eg.eg.SetLimit(n) +} + +// Wait waits for all goroutines to finish and returns the first error that occurred. +func (eg *ErrGroup) Wait() error { + return eg.eg.Wait() +} + +// Go runs the given function in a goroutine, handling panics by logging them. +// +// This function is a panic-handling wrapper for the "go" keyword. +func Go(f func(), logger *zap.Logger) { + go func() { + defer func() { + if r := recover(); r != nil { + stack := debug.Stack() + + if logger != nil { + logger.DPanic(goroutinePanicked, zap.Any("panic", r), zap.String("stack", string(stack))) + } else { + log.Printf("[fallback logger] %s: %s\n%s", goroutinePanicked, r, string(stack)) + } + } + }() + + f() + }() +} diff --git a/client/pkg/panichandler/panichandler_test.go b/client/pkg/panichandler/panichandler_test.go new file mode 100644 index 00000000..7a463518 --- /dev/null +++ b/client/pkg/panichandler/panichandler_test.go @@ -0,0 +1,113 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package panichandler_test + +import ( + "errors" + "runtime" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" + + "github.com/siderolabs/omni/client/pkg/panichandler" +) + +func TestGoNoPanic(t *testing.T) { + logObserverCore, observedLogs := observer.New(zapcore.InfoLevel) + logger := zap.New(logObserverCore) + + var wg sync.WaitGroup + + wg.Add(1) + + panichandler.Go(func() { + wg.Done() + }, logger) + + wg.Wait() + + assert.Empty(t, observedLogs.All()) +} + +func TestGoLogPanic(t *testing.T) { + currentFile, currentFunc := trace() + + logObserverCore, observedLogs := observer.New(zapcore.InfoLevel) + logger := zap.New(logObserverCore) + + panichandler.Go(func() { + panic("test") + }, logger) + + require.EventuallyWithT(t, func(collect *assert.CollectT) { + if assert.Equal(collect, 1, observedLogs.Len()) { + loggedEntry := observedLogs.All()[0] + stack := loggedEntry.ContextMap()["stack"] + + t.Logf("log msg: %s stack: %s", loggedEntry.Message, stack) + + assert.Equal(collect, zapcore.DPanicLevel, loggedEntry.Level) + assert.Contains(collect, loggedEntry.Message, panichandler.ErrPanic.Error()) + + // assert stack trace + assert.Contains(collect, stack, currentFunc) + assert.Contains(collect, stack, currentFile) + } + }, 1*time.Second, 10*time.Millisecond) +} + +func TestRunErrFErrUnchanged(t *testing.T) { + expectedErr := errors.New("test error") + + eg := panichandler.NewErrGroup() + + eg.Go(func() error { + return expectedErr + }) + + err := eg.Wait() + + assert.Equal(t, expectedErr, err) +} + +func TestRunErrFHandlePanic(t *testing.T) { + currentFile, currentFunc := trace() + + eg := panichandler.NewErrGroup() + + eg.Go(func() error { + return nil // no err, all good + }) + + eg.Go(func() error { + panic("test") + }) + + err := eg.Wait() + + t.Logf("error: %v", err) + + assert.ErrorIs(t, err, panichandler.ErrPanic) + + // assert stack trace + assert.ErrorContains(t, err, currentFunc) + assert.ErrorContains(t, err, currentFile) +} + +// trace returns the file and function name of the caller. +func trace() (file, function string) { + pc := make([]uintptr, 15) + n := runtime.Callers(2, pc) + frames := runtime.CallersFrames(pc[:n]) + frame, _ := frames.Next() + + return frame.File, frame.Function +} diff --git a/cmd/omni/main.go b/cmd/omni/main.go index eaaea984..4f683c65 100644 --- a/cmd/omni/main.go +++ b/cmd/omni/main.go @@ -29,6 +29,7 @@ import ( "github.com/siderolabs/omni/client/pkg/constants" authres "github.com/siderolabs/omni/client/pkg/omni/resources/auth" omnires "github.com/siderolabs/omni/client/pkg/omni/resources/omni" + "github.com/siderolabs/omni/client/pkg/panichandler" "github.com/siderolabs/omni/internal/backend" "github.com/siderolabs/omni/internal/backend/discovery" "github.com/siderolabs/omni/internal/backend/dns" @@ -122,15 +123,17 @@ var rootCmd = &cobra.Command{ defer stop() // do not use signal.NotifyContext as it doesn't support any ways to log the received signal - go func() { + panichandler.Go(func() { s := <-signals logger.Warn("signal received, stopping Omni", zap.String("signal", s.String())) stop() - }() + }, logger) - go runDebugServer(ctx, logger) + panichandler.Go(func() { + runDebugServer(ctx, logger) + }, logger) // this global context propagates into all controllers and any other background activities ctx = actor.MarkContextAsInternalActor(ctx) diff --git a/go.mod b/go.mod index 70a32c4f..f6da4c17 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/siderolabs/omni -go 1.22.3 +go 1.22.4 replace ( // use nested module diff --git a/internal/backend/grpc/management.go b/internal/backend/grpc/management.go index d824ed5a..dbadd36f 100644 --- a/internal/backend/grpc/management.go +++ b/internal/backend/grpc/management.go @@ -46,6 +46,7 @@ import ( authres "github.com/siderolabs/omni/client/pkg/omni/resources/auth" omnires "github.com/siderolabs/omni/client/pkg/omni/resources/omni" ctlcfg "github.com/siderolabs/omni/client/pkg/omnictl/config" + "github.com/siderolabs/omni/client/pkg/panichandler" "github.com/siderolabs/omni/internal/backend/dns" "github.com/siderolabs/omni/internal/backend/grpc/router" "github.com/siderolabs/omni/internal/backend/imagefactory" @@ -239,11 +240,11 @@ func (s *managementServer) MachineLogs(request *management.MachineLogsRequest, r defer cancel() - go func() { + panichandler.Go(func() { // connection closed, stop reading <-response.Context().Done() cancel() - }() + }, s.logger) for { line, err := logReader.ReadLine() @@ -813,9 +814,9 @@ func (s *managementServer) KubernetesSyncManifests(req *management.KubernetesSyn errCh := make(chan error, 1) synCh := make(chan manifests.SyncResult) - go func() { + panichandler.Go(func() { errCh <- manifests.Sync(ctx, bootstrapManifests, cfg, req.DryRun, synCh) - }() + }, s.logger) var updatedManifests []manifests.Manifest @@ -857,9 +858,9 @@ syncLoop: rolloutCh := make(chan manifests.RolloutProgress) - go func() { + panichandler.Go(func() { errCh <- manifests.WaitForRollout(ctx, cfg, updatedManifests, rolloutCh) - }() + }, s.logger) rolloutLoop: for { diff --git a/internal/backend/grpc/resource.go b/internal/backend/grpc/resource.go index 35589e7e..2637afdf 100644 --- a/internal/backend/grpc/resource.go +++ b/internal/backend/grpc/resource.go @@ -16,7 +16,6 @@ import ( "github.com/cosi-project/runtime/pkg/state" gateway "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" "github.com/siderolabs/go-api-signature/pkg/message" - "golang.org/x/sync/errgroup" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" @@ -25,6 +24,7 @@ import ( "github.com/siderolabs/omni/client/api/common" "github.com/siderolabs/omni/client/api/omni/resources" + "github.com/siderolabs/omni/client/pkg/panichandler" "github.com/siderolabs/omni/internal/backend/grpc/router" "github.com/siderolabs/omni/internal/backend/runtime" ) @@ -172,7 +172,7 @@ func (s *ResourceServer) Watch(in *resources.WatchRequest, serv resources.Resour } events := make(chan runtime.WatchResponse) - eg, ctx := errgroup.WithContext(ctx) + eg, ctx := panichandler.ErrGroupWithContext(ctx) if in.TailEvents != 0 { opts = append(opts, runtime.WithTailEvents(int(in.TailEvents))) diff --git a/internal/backend/grpc/support.go b/internal/backend/grpc/support.go index f5bb96a2..8f754ed9 100644 --- a/internal/backend/grpc/support.go +++ b/internal/backend/grpc/support.go @@ -23,13 +23,13 @@ import ( "github.com/siderolabs/go-talos-support/support/bundle" "github.com/siderolabs/go-talos-support/support/collectors" "github.com/siderolabs/talos/pkg/machinery/client" - "golang.org/x/sync/errgroup" "gopkg.in/yaml.v3" "k8s.io/client-go/kubernetes" "github.com/siderolabs/omni/client/api/omni/management" "github.com/siderolabs/omni/client/pkg/omni/resources/omni" "github.com/siderolabs/omni/client/pkg/omni/resources/siderolink" + "github.com/siderolabs/omni/client/pkg/panichandler" "github.com/siderolabs/omni/internal/backend/runtime" kubernetesruntime "github.com/siderolabs/omni/internal/backend/runtime/kubernetes" "github.com/siderolabs/omni/internal/pkg/auth" @@ -127,7 +127,7 @@ func (s *managementServer) GetSupportBundle(req *management.GetSupportBundleRequ cols = append(cols, talosCollectors...) - var eg errgroup.Group + eg := panichandler.NewErrGroup() eg.Go(func() error { for p := range progress { diff --git a/internal/backend/runtime/kubernetes/kubernetes.go b/internal/backend/runtime/kubernetes/kubernetes.go index b15dc108..4083bdf4 100644 --- a/internal/backend/runtime/kubernetes/kubernetes.go +++ b/internal/backend/runtime/kubernetes/kubernetes.go @@ -25,7 +25,6 @@ import ( "github.com/hashicorp/golang-lru/v2/expirable" "github.com/prometheus/client_golang/prometheus" "github.com/siderolabs/gen/channel" - "golang.org/x/sync/errgroup" "golang.org/x/sync/singleflight" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -40,6 +39,7 @@ import ( "github.com/siderolabs/omni/client/api/omni/resources" "github.com/siderolabs/omni/client/pkg/omni/resources/k8s" "github.com/siderolabs/omni/client/pkg/omni/resources/omni" + "github.com/siderolabs/omni/client/pkg/panichandler" pkgruntime "github.com/siderolabs/omni/client/pkg/runtime" "github.com/siderolabs/omni/internal/backend/oidc/external" "github.com/siderolabs/omni/internal/backend/runtime" @@ -473,7 +473,7 @@ func (w *Watch) run(ctx context.Context, client *Client) error { errCh := make(chan error, 1) - eg, ctx := errgroup.WithContext(ctx) + eg, ctx := panichandler.ErrGroupWithContext(ctx) if err := informer.Informer().SetWatchErrorHandler(func(_ *toolscache.Reflector, e error) { channel.SendWithContext(ctx, errCh, e) diff --git a/internal/backend/runtime/omni/controllers/omni/etcd_backup.go b/internal/backend/runtime/omni/controllers/omni/etcd_backup.go index 47d66299..8df02338 100644 --- a/internal/backend/runtime/omni/controllers/omni/etcd_backup.go +++ b/internal/backend/runtime/omni/controllers/omni/etcd_backup.go @@ -21,12 +21,12 @@ import ( "github.com/siderolabs/gen/containers" machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" "go.uber.org/zap" - "golang.org/x/sync/errgroup" "google.golang.org/grpc" "google.golang.org/protobuf/types/known/timestamppb" "github.com/siderolabs/omni/client/api/omni/specs" "github.com/siderolabs/omni/client/pkg/omni/resources/omni" + "github.com/siderolabs/omni/client/pkg/panichandler" "github.com/siderolabs/omni/internal/backend/runtime/omni/controllers/omni/etcdbackup" "github.com/siderolabs/omni/internal/backend/runtime/omni/controllers/omni/etcdbackup/store" ) @@ -160,7 +160,7 @@ func (ctrl *EtcdBackupController) run(ctx context.Context, r controller.Runtime, return nil } - var eg errgroup.Group + eg := panichandler.NewErrGroup() eg.SetLimit(ctrl.parallel) diff --git a/internal/backend/runtime/omni/controllers/omni/internal/etcdbackup/crypt/crypt.go b/internal/backend/runtime/omni/controllers/omni/internal/etcdbackup/crypt/crypt.go index 61b11e92..9f6a7d0b 100644 --- a/internal/backend/runtime/omni/controllers/omni/internal/etcdbackup/crypt/crypt.go +++ b/internal/backend/runtime/omni/controllers/omni/internal/etcdbackup/crypt/crypt.go @@ -11,8 +11,7 @@ import ( "fmt" "io" - "golang.org/x/sync/errgroup" - + "github.com/siderolabs/omni/client/pkg/panichandler" "github.com/siderolabs/omni/internal/backend/runtime/omni/controllers/omni/etcdbackup" ) @@ -32,7 +31,7 @@ func (c *Store) Upload(ctx context.Context, descr etcdbackup.Description, r io.R ctx, cancel := context.WithCancel(ctx) defer cancel() - eg, ctx := errgroup.WithContext(ctx) + eg, ctx := panichandler.ErrGroupWithContext(ctx) reader, writer := io.Pipe() diff --git a/internal/backend/runtime/omni/controllers/omni/internal/task/clustermachine/identity.go b/internal/backend/runtime/omni/controllers/omni/internal/task/clustermachine/identity.go index ec4bf56a..c61aad1a 100644 --- a/internal/backend/runtime/omni/controllers/omni/internal/task/clustermachine/identity.go +++ b/internal/backend/runtime/omni/controllers/omni/internal/task/clustermachine/identity.go @@ -36,6 +36,7 @@ import ( "github.com/siderolabs/omni/client/pkg/omni/resources" "github.com/siderolabs/omni/client/pkg/omni/resources/omni" + "github.com/siderolabs/omni/client/pkg/panichandler" "github.com/siderolabs/omni/internal/backend/runtime/talos" ) @@ -101,7 +102,7 @@ func (spec IdentityCollectorTaskSpec) Equal(other IdentityCollectorTaskSpec) boo // RunTask runs the identity collector task. // //nolint:gocyclo,cyclop -func (spec IdentityCollectorTaskSpec) RunTask(ctx context.Context, _ *zap.Logger, notify IdentityCollectorChan) error { +func (spec IdentityCollectorTaskSpec) RunTask(ctx context.Context, logger *zap.Logger, notify IdentityCollectorChan) error { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -162,11 +163,11 @@ func (spec IdentityCollectorTaskSpec) RunTask(ctx context.Context, _ *zap.Logger if runLegacyEtcdMemberIDCollector { wg.Add(1) - go func() { + panichandler.Go(func() { defer wg.Done() spec.runLegacyEtcdMemberIDCollector(ctx, client, watchCh) //nolint:errcheck - }() + }, logger) } for { diff --git a/internal/backend/runtime/omni/controllers/omni/internal/task/task.go b/internal/backend/runtime/omni/controllers/omni/internal/task/task.go index aa3bc707..8086e4e1 100644 --- a/internal/backend/runtime/omni/controllers/omni/internal/task/task.go +++ b/internal/backend/runtime/omni/controllers/omni/internal/task/task.go @@ -14,6 +14,8 @@ import ( "github.com/cenkalti/backoff/v4" "go.uber.org/zap" + + "github.com/siderolabs/omni/client/pkg/panichandler" ) // ID is a task ID. @@ -55,11 +57,11 @@ func (task *Task[T, S]) Start(ctx context.Context) { ctx, task.cancel = context.WithCancel(ctx) - go func() { + panichandler.Go(func() { defer task.wg.Done() task.runWithRestarts(ctx) - }() + }, task.logger) } func (task *Task[T, S]) runWithRestarts(ctx context.Context) { diff --git a/internal/backend/runtime/omni/controllers/omni/kubernetes_status.go b/internal/backend/runtime/omni/controllers/omni/kubernetes_status.go index b42b476c..92ce0f64 100644 --- a/internal/backend/runtime/omni/controllers/omni/kubernetes_status.go +++ b/internal/backend/runtime/omni/controllers/omni/kubernetes_status.go @@ -39,6 +39,7 @@ import ( "github.com/siderolabs/omni/client/api/omni/specs" "github.com/siderolabs/omni/client/pkg/omni/resources" "github.com/siderolabs/omni/client/pkg/omni/resources/omni" + "github.com/siderolabs/omni/client/pkg/panichandler" "github.com/siderolabs/omni/internal/backend/runtime" "github.com/siderolabs/omni/internal/backend/runtime/kubernetes" "github.com/siderolabs/omni/internal/pkg/config" @@ -533,19 +534,19 @@ func (ctrl *KubernetesStatusController) startWatcher(ctx context.Context, logger w.nodeFactory.Start(ctx.Done()) w.podFactory.Start(ctx.Done()) - go func() { + panichandler.Go(func() { w.nodeFactory.WaitForCacheSync(ctx.Done()) w.nodesSynced.Store(true) w.nodesSync(ctx, notifyCh) - }() + }, logger) - go func() { + panichandler.Go(func() { w.podFactory.WaitForCacheSync(ctx.Done()) w.podsSynced.Store(true) w.podsSync(ctx, notifyCh) - }() + }, logger) if config.Config.WorkloadProxying.Enabled { w.serviceFactory = informers.NewSharedInformerFactory(w.client.Clientset(), 0) @@ -562,12 +563,12 @@ func (ctrl *KubernetesStatusController) startWatcher(ctx context.Context, logger w.serviceFactory.Start(ctx.Done()) - go func() { + panichandler.Go(func() { w.serviceFactory.WaitForCacheSync(ctx.Done()) w.servicesSynced.Store(true) w.servicesSync(ctx, notifyCh) - }() + }, logger) } ctrl.watchers[cluster] = w diff --git a/internal/backend/runtime/omni/controllers/omni/kubernetes_upgrade_manifest_status.go b/internal/backend/runtime/omni/controllers/omni/kubernetes_upgrade_manifest_status.go index 3e03b10c..9e570a70 100644 --- a/internal/backend/runtime/omni/controllers/omni/kubernetes_upgrade_manifest_status.go +++ b/internal/backend/runtime/omni/controllers/omni/kubernetes_upgrade_manifest_status.go @@ -27,6 +27,7 @@ import ( "github.com/siderolabs/omni/client/api/omni/specs" "github.com/siderolabs/omni/client/pkg/omni/resources" "github.com/siderolabs/omni/client/pkg/omni/resources/omni" + "github.com/siderolabs/omni/client/pkg/panichandler" "github.com/siderolabs/omni/internal/backend/runtime" "github.com/siderolabs/omni/internal/backend/runtime/kubernetes" "github.com/siderolabs/omni/internal/backend/runtime/omni/controllers/helpers" @@ -208,9 +209,9 @@ func NewKubernetesUpgradeManifestStatusController() *KubernetesUpgradeManifestSt errCh := make(chan error, 1) resultCh := make(chan manifests.SyncResult) - go func() { + panichandler.Go(func() { errCh <- manifests.Sync(ctx, bootstrapManifests, cfg, true, resultCh) - }() + }, logger) for { select { diff --git a/internal/backend/runtime/omni/external/state.go b/internal/backend/runtime/omni/external/state.go index d31aed84..97259fa3 100644 --- a/internal/backend/runtime/omni/external/state.go +++ b/internal/backend/runtime/omni/external/state.go @@ -18,10 +18,12 @@ import ( "github.com/cosi-project/runtime/pkg/state" "github.com/siderolabs/gen/channel" "github.com/siderolabs/gen/pair/ordered" + "go.uber.org/zap" "google.golang.org/protobuf/types/known/timestamppb" "github.com/siderolabs/omni/client/pkg/omni/resources" "github.com/siderolabs/omni/client/pkg/omni/resources/omni" + "github.com/siderolabs/omni/client/pkg/panichandler" "github.com/siderolabs/omni/internal/backend/runtime/omni/controllers/omni/etcdbackup/store" ) @@ -31,6 +33,7 @@ import ( type State struct { CoreState state.CoreState StoreFactory store.Factory + Logger *zap.Logger } // Get implements [state.CoreState] interface. @@ -181,7 +184,7 @@ func (s *State) WatchKind(ctx context.Context, kind resource.Kind, ch chan<- sta return makeUnsupportedError(err) } - go func() { + panichandler.Go(func() { list, err := s.List(ctx, kind, convertedOpts...) if err != nil { channel.SendWithContext(ctx, ch, state.Event{ @@ -204,7 +207,7 @@ func (s *State) WatchKind(ctx context.Context, kind resource.Kind, ch chan<- sta if !channel.SendWithContext(ctx, ch, state.Event{Type: state.Bootstrapped}) { return } - }() + }, s.Logger) return nil } diff --git a/internal/backend/runtime/omni/external/state_test.go b/internal/backend/runtime/omni/external/state_test.go index a701d35f..9ad9def1 100644 --- a/internal/backend/runtime/omni/external/state_test.go +++ b/internal/backend/runtime/omni/external/state_test.go @@ -20,6 +20,7 @@ import ( "github.com/siderolabs/gen/pair" "github.com/siderolabs/gen/xtesting/check" "github.com/stretchr/testify/assert" + "go.uber.org/zap/zaptest" "github.com/siderolabs/omni/client/pkg/omni/resources" "github.com/siderolabs/omni/client/pkg/omni/resources/omni" @@ -133,6 +134,7 @@ func TestStateList(t *testing.T) { s := &external.State{ CoreState: tt.coreState, StoreFactory: tt.storeFactory, + Logger: zaptest.NewLogger(t), } list, err := s.List(context.Background(), tt.kind, tt.opts...) @@ -229,6 +231,7 @@ func TestStateGet(t *testing.T) { s := &external.State{ CoreState: tt.coreState, StoreFactory: tt.storeFactory, + Logger: zaptest.NewLogger(t), } backup, err := s.Get(context.Background(), tt.pointer, tt.opts...) diff --git a/internal/backend/runtime/omni/pkg/check/etcd.go b/internal/backend/runtime/omni/pkg/check/etcd.go index b0ac7b20..51ae9776 100644 --- a/internal/backend/runtime/omni/pkg/check/etcd.go +++ b/internal/backend/runtime/omni/pkg/check/etcd.go @@ -337,7 +337,7 @@ func getMemberState(ctx context.Context, talosConfig *omni.TalosConfig, clusterM if err != nil { status.Error = err.Error() - return status, nil //nolint:nilerr + return status, nil } for _, info := range list { diff --git a/internal/backend/runtime/omni/state.go b/internal/backend/runtime/omni/state.go index 3be911cb..6e9906b0 100644 --- a/internal/backend/runtime/omni/state.go +++ b/internal/backend/runtime/omni/state.go @@ -65,6 +65,7 @@ func NewState(ctx context.Context, params *config.Params, logger *zap.Logger, me return &external.State{ CoreState: primaryStorageCoreState, StoreFactory: storeFactory, + Logger: logger, } default: return primaryStorageCoreState diff --git a/internal/backend/runtime/omni/state_etcd.go b/internal/backend/runtime/omni/state_etcd.go index e09daa9e..ba2289db 100644 --- a/internal/backend/runtime/omni/state_etcd.go +++ b/internal/backend/runtime/omni/state_etcd.go @@ -31,6 +31,7 @@ import ( "google.golang.org/grpc" "github.com/siderolabs/omni/client/pkg/constants" + "github.com/siderolabs/omni/client/pkg/panichandler" "github.com/siderolabs/omni/internal/backend/logging" "github.com/siderolabs/omni/internal/backend/runtime/keyprovider" "github.com/siderolabs/omni/internal/pkg/config" @@ -183,7 +184,7 @@ func getEmbeddedEtcdClient(ctx context.Context, params *config.EtcdParams, logge return fmt.Errorf("failed to start embedded etcd: %w", err) } - go func() { + panichandler.Go(func() { for etcdErr := range embeddedServer.Err() { if etcdErr != nil { logger.Error("embedded etcd error", zap.Error(etcdErr)) @@ -191,7 +192,7 @@ func getEmbeddedEtcdClient(ctx context.Context, params *config.EtcdParams, logge cancel() } } - }() + }, logger) // give etcd some time to start timer := time.NewTimer(15 * time.Second) diff --git a/internal/backend/runtime/omni/state_etcd_election.go b/internal/backend/runtime/omni/state_etcd_election.go index 37cdfcd7..2310c995 100644 --- a/internal/backend/runtime/omni/state_etcd_election.go +++ b/internal/backend/runtime/omni/state_etcd_election.go @@ -16,6 +16,7 @@ import ( "go.etcd.io/etcd/client/v3/concurrency" "go.uber.org/zap" + "github.com/siderolabs/omni/client/pkg/panichandler" "github.com/siderolabs/omni/internal/backend/logging" ) @@ -37,9 +38,9 @@ func etcdElections(ctx context.Context, client *clientv3.Client, electionKey str campaignErrCh := make(chan error) - go func() { + panichandler.Go(func() { campaignErrCh <- election.Campaign(ctx, campaignKey) - }() + }, logger) logger.Info("running the etcd election campaign") @@ -78,7 +79,7 @@ campaignLoop: ctx, cancel := context.WithCancel(ctx) defer cancel() - go func() { + panichandler.Go(func() { observe := election.Observe(ctx) observeLoop: @@ -102,7 +103,7 @@ campaignLoop: } } } - }() + }, logger) return f(ctx, client) } diff --git a/internal/backend/runtime/omni/virtual/computed.go b/internal/backend/runtime/omni/virtual/computed.go index 34c61e22..46bcdbbd 100644 --- a/internal/backend/runtime/omni/virtual/computed.go +++ b/internal/backend/runtime/omni/virtual/computed.go @@ -16,6 +16,8 @@ import ( "github.com/cosi-project/runtime/pkg/state/impl/namespaced" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" + + "github.com/siderolabs/omni/client/pkg/panichandler" ) // Computed is a virtual state implementation which provides virtual resources which are computed on the fly @@ -25,6 +27,7 @@ type Computed struct { watchScheduler *DedupScheduler activeWatches *prometheus.GaugeVec resolveID ProducerIDTransformer + logger *zap.Logger } // ProducerIDTransformer maps the incoming resource id into some other id. @@ -53,6 +56,7 @@ func NewComputed(resourceType string, factory ProducerFactory, resolveID Produce "type", "id", }), + logger: logger, } } @@ -114,13 +118,13 @@ func (st *Computed) Watch(ctx context.Context, ptr resource.Pointer, c chan<- st activeWatches.Inc() - go func() { + panichandler.Go(func() { <-ctx.Done() activeWatches.Dec() st.watchScheduler.Stop(ptr) - }() + }, st.logger) return nil } diff --git a/internal/backend/runtime/omni/virtual/pkg/producers/kubernetes_usage.go b/internal/backend/runtime/omni/virtual/pkg/producers/kubernetes_usage.go index 27f928e7..4b8eb61e 100644 --- a/internal/backend/runtime/omni/virtual/pkg/producers/kubernetes_usage.go +++ b/internal/backend/runtime/omni/virtual/pkg/producers/kubernetes_usage.go @@ -23,6 +23,7 @@ import ( "github.com/siderolabs/omni/client/api/omni/specs" "github.com/siderolabs/omni/client/pkg/omni/resources/omni" "github.com/siderolabs/omni/client/pkg/omni/resources/virtual" + "github.com/siderolabs/omni/client/pkg/panichandler" "github.com/siderolabs/omni/internal/backend/logging" "github.com/siderolabs/omni/internal/backend/runtime" "github.com/siderolabs/omni/internal/backend/runtime/kubernetes" @@ -168,7 +169,7 @@ func (ku *KubernetesUsage) Start() error { ku.wg.Add(2) - go func() { + panichandler.Go(func() { defer ku.wg.Done() ku.factory.WaitForCacheSync(ku.stopCh) @@ -183,9 +184,9 @@ func (ku *KubernetesUsage) Start() error { synced.Store(true) sync() - }() + }, ku.logger) - go func() { + panichandler.Go(func() { defer ku.wg.Done() <-ku.stopCh @@ -193,7 +194,7 @@ func (ku *KubernetesUsage) Start() error { synced.Store(false) ku.factory.Shutdown() - }() + }, ku.logger) return nil } diff --git a/internal/backend/runtime/proxy_runtime.go b/internal/backend/runtime/proxy_runtime.go index 0ed5c383..22d47df2 100644 --- a/internal/backend/runtime/proxy_runtime.go +++ b/internal/backend/runtime/proxy_runtime.go @@ -15,9 +15,9 @@ import ( "github.com/siderolabs/gen/channel" "github.com/siderolabs/gen/pair" "github.com/siderolabs/gen/pair/ordered" - "golang.org/x/sync/errgroup" "github.com/siderolabs/omni/client/api/omni/resources" + "github.com/siderolabs/omni/client/pkg/panichandler" "github.com/siderolabs/omni/client/pkg/runtime" ) @@ -27,14 +27,14 @@ func (p *proxyRuntime) Watch(ctx context.Context, responses chan<- WatchResponse ctx, cancel := context.WithCancel(ctx) defer cancel() - var group errgroup.Group + eg := panichandler.NewErrGroup() opts := NewQueryOptions(option...) cmp := MakeWatchResponseComparator(opts.SortField, opts.SortDescending) ch := make(chan WatchResponse) produce := watchResponseProducer(responses, opts, cmp) - group.Go(func() error { + eg.Go(func() error { defer cancel() slc, err := takeSorted(ctx, ch, cmp) @@ -60,13 +60,13 @@ func (p *proxyRuntime) Watch(ctx context.Context, responses chan<- WatchResponse } }) - group.Go(func() error { + eg.Go(func() error { defer cancel() return p.Runtime.Watch(ctx, ch, option...) }) - return group.Wait() + return eg.Wait() } func watchResponseProducer( diff --git a/internal/backend/server.go b/internal/backend/server.go index d59b6961..255a4e77 100644 --- a/internal/backend/server.go +++ b/internal/backend/server.go @@ -56,6 +56,7 @@ import ( "github.com/siderolabs/omni/client/pkg/omni/resources" authres "github.com/siderolabs/omni/client/pkg/omni/resources/auth" omnires "github.com/siderolabs/omni/client/pkg/omni/resources/omni" + "github.com/siderolabs/omni/client/pkg/panichandler" "github.com/siderolabs/omni/internal/backend/debug" "github.com/siderolabs/omni/internal/backend/dns" "github.com/siderolabs/omni/internal/backend/factory" @@ -530,7 +531,7 @@ func (s *Server) runMachineAPI(ctx context.Context) error { ) }) - grpcutil.RunServer(groupCtx, server, lis, eg) + grpcutil.RunServer(groupCtx, server, lis, eg, s.logger) return eg.Wait() } @@ -859,7 +860,9 @@ func runServer(ctx context.Context, srv *server, logger *zap.Logger) error { errCh := make(chan error, 1) - go func() { errCh <- srv.ListenAndServe() }() + panichandler.Go(func() { + errCh <- srv.ListenAndServe() + }, logger) select { case err := <-errCh: @@ -921,7 +924,7 @@ func runLocalResourceServer(ctx context.Context, st state.CoreState, serverOptio logger.Info("starting local resource server") - grpcutil.RunServer(ctx, grpcServer, listener, eg) + grpcutil.RunServer(ctx, grpcServer, listener, eg, logger) return nil } @@ -971,7 +974,9 @@ func runGRPCServer(ctx context.Context, server *grpc.Server, transport *memconn. errCh := make(chan error, 1) - go func() { errCh <- server.Serve(grpcListener) }() + panichandler.Go(func() { + errCh <- server.Serve(grpcListener) + }, logger) select { case err := <-errCh: diff --git a/internal/pkg/errgroup/errgroup.go b/internal/pkg/errgroup/errgroup.go index 5c74f1aa..19815eca 100644 --- a/internal/pkg/errgroup/errgroup.go +++ b/internal/pkg/errgroup/errgroup.go @@ -11,7 +11,7 @@ import ( "fmt" "runtime/debug" - "golang.org/x/sync/errgroup" + "github.com/siderolabs/omni/client/pkg/panichandler" ) // EGroup defines common interface for Group and x/sync/errgroup.Group. @@ -23,7 +23,7 @@ type EGroup interface { // Group is wrapper around Go's x/sync/errgroup.Group. It's not a drop-in replacement for it, because // it requires initialization with WithContext. type Group struct { - group *errgroup.Group + group EGroup ctx context.Context //nolint:containedctx } @@ -33,7 +33,7 @@ type Group struct { // returns or the first time Wait returns, whichever occurs // first. func WithContext(ctx context.Context) (*Group, context.Context) { - withContext, newCtx := errgroup.WithContext(ctx) + withContext, newCtx := panichandler.ErrGroupWithContext(ctx) return &Group{group: withContext, ctx: newCtx}, newCtx } diff --git a/internal/pkg/grpcutil/server.go b/internal/pkg/grpcutil/server.go index 8a2d8d76..7e22f47c 100644 --- a/internal/pkg/grpcutil/server.go +++ b/internal/pkg/grpcutil/server.go @@ -12,13 +12,15 @@ import ( "net" "time" + "go.uber.org/zap" "google.golang.org/grpc" + "github.com/siderolabs/omni/client/pkg/panichandler" "github.com/siderolabs/omni/internal/pkg/errgroup" ) // RunServer starts gRPC server on top of the provided listener, stops it when the context is done. -func RunServer(ctx context.Context, server *grpc.Server, lis net.Listener, eg *errgroup.Group) { +func RunServer(ctx context.Context, server *grpc.Server, lis net.Listener, eg *errgroup.Group, logger *zap.Logger) { eg.Go(func() error { err := server.Serve(lis) if !errors.Is(err, grpc.ErrServerStopped) { @@ -29,13 +31,13 @@ func RunServer(ctx context.Context, server *grpc.Server, lis net.Listener, eg *e }) eg.Go(func() error { - serverGracefulStop(server, ctx) + serverGracefulStop(server, ctx, logger) return nil }) } -func serverGracefulStop(server *grpc.Server, ctx context.Context) { //nolint:revive +func serverGracefulStop(server *grpc.Server, ctx context.Context, logger *zap.Logger) { //nolint:revive <-ctx.Done() stopped := make(chan struct{}) @@ -43,10 +45,10 @@ func serverGracefulStop(server *grpc.Server, ctx context.Context) { //nolint:rev shutdownCtx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - go func() { + panichandler.Go(func() { server.GracefulStop() close(stopped) - }() + }, logger) select { case <-shutdownCtx.Done(): diff --git a/internal/pkg/logreceiver/logreceiver.go b/internal/pkg/logreceiver/logreceiver.go index 49f4bb86..6e32ab2b 100644 --- a/internal/pkg/logreceiver/logreceiver.go +++ b/internal/pkg/logreceiver/logreceiver.go @@ -17,6 +17,8 @@ import ( "github.com/siderolabs/gen/containers" "go.uber.org/zap" + + "github.com/siderolabs/omni/client/pkg/panichandler" ) // Server implements TCP server to receive JSON logs. It is similar to the one in Talos, except it doesn't try to parse @@ -75,11 +77,11 @@ func (srv *Server) Serve() error { srv.wg.Add(1) srv.m.Set(remoteAddress, conn) - go func() { + panichandler.Go(func() { defer srv.wg.Done() srv.handler.HandleConn(remoteAddr, conn) srv.m.Remove(remoteAddress) - }() + }, srv.logger) } } diff --git a/internal/pkg/siderolink/manager.go b/internal/pkg/siderolink/manager.go index d78cd8f4..1895afa9 100644 --- a/internal/pkg/siderolink/manager.go +++ b/internal/pkg/siderolink/manager.go @@ -412,13 +412,13 @@ func (manager *Manager) startEventsGRPC(ctx context.Context, eg *errgroup.Group, }) eventsapi.RegisterEventSinkServiceServer(server, sink) - grpcutil.RunServer(ctx, server, listener, eg) + grpcutil.RunServer(ctx, server, listener, eg, manager.logger) } func (manager *Manager) startTrustdGRPC(ctx context.Context, eg *errgroup.Group, listener net.Listener, serverAddr netip.Prefix) { server := trustd.NewServer(manager.logger, manager.state, serverAddr.Addr().AsSlice()) //nolint:contextcheck - grpcutil.RunServer(ctx, server, listener, eg) + grpcutil.RunServer(ctx, server, listener, eg, manager.logger) } func (manager *Manager) startLogServer(ctx context.Context, eg *errgroup.Group, serverAddr netip.Prefix, logServerPort string) error { diff --git a/internal/pkg/siderolink/siderolink_test.go b/internal/pkg/siderolink/siderolink_test.go index 55f1bcbf..a128a306 100644 --- a/internal/pkg/siderolink/siderolink_test.go +++ b/internal/pkg/siderolink/siderolink_test.go @@ -148,7 +148,7 @@ func (suite *SiderolinkSuite) startManager(params sideromanager.Params) { ) }) - grpcutil.RunServer(groupCtx, server, lis, eg) + grpcutil.RunServer(groupCtx, server, lis, eg, zaptest.NewLogger(suite.T())) suite.Require().NoError(eg.Wait()) }() diff --git a/internal/version/data/tag b/internal/version/data/tag index 52e9df6a..6b1bb122 100644 --- a/internal/version/data/tag +++ b/internal/version/data/tag @@ -1 +1 @@ -v0.38.0-beta.0 +v0.38.0-beta.0 \ No newline at end of file