diff --git a/cmd/traefik/logger.go b/cmd/traefik/logger.go index a8b609e4b..5e84e9118 100644 --- a/cmd/traefik/logger.go +++ b/cmd/traefik/logger.go @@ -1,6 +1,7 @@ package main import ( + "context" "errors" "fmt" "io" @@ -22,7 +23,7 @@ func init() { zerolog.SetGlobalLevel(zerolog.ErrorLevel) } -func setupLogger(staticConfiguration *static.Configuration) error { +func setupLogger(ctx context.Context, staticConfiguration *static.Configuration) error { // Validate that the experimental flag is set up at this point, // rather than validating the static configuration before the setupLogger call. // This ensures that validation messages are not logged using an un-configured logger. @@ -39,16 +40,16 @@ func setupLogger(staticConfiguration *static.Configuration) error { zerolog.SetGlobalLevel(logLevel) // create logger - logCtx := zerolog.New(w).With().Timestamp() + logger := zerolog.New(w).With().Timestamp() if logLevel <= zerolog.DebugLevel { - logCtx = logCtx.Caller() + logger = logger.Caller() } - log.Logger = logCtx.Logger().Level(logLevel) + log.Logger = logger.Logger().Level(logLevel) if staticConfiguration.Log != nil && staticConfiguration.Log.OTLP != nil { var err error - log.Logger, err = logs.SetupOTelLogger(log.Logger, staticConfiguration.Log.OTLP) + log.Logger, err = logs.SetupOTelLogger(ctx, log.Logger, staticConfiguration.Log.OTLP) if err != nil { return fmt.Errorf("setting up OpenTelemetry logger: %w", err) } diff --git a/cmd/traefik/traefik.go b/cmd/traefik/traefik.go index 93e319f6a..dc80319e3 100644 --- a/cmd/traefik/traefik.go +++ b/cmd/traefik/traefik.go @@ -90,7 +90,10 @@ Complete documentation is available at https://traefik.io`, } func runCmd(staticConfiguration *static.Configuration) error { - if err := setupLogger(staticConfiguration); err != nil { + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + if err := setupLogger(ctx, staticConfiguration); err != nil { return fmt.Errorf("setting up logger: %w", err) } @@ -123,8 +126,6 @@ func runCmd(staticConfiguration *static.Configuration) error { return err } - ctx, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) - if staticConfiguration.Ping != nil { staticConfiguration.Ping.WithContext(ctx) } @@ -210,8 +211,8 @@ func setupServer(staticConfiguration *static.Configuration) (*server.Server, err } } metricsRegistry := metrics.NewMultiRegistry(metricRegistries) - accessLog := setupAccessLog(staticConfiguration.AccessLog) - tracer, tracerCloser := setupTracing(staticConfiguration.Tracing) + accessLog := setupAccessLog(ctx, staticConfiguration.AccessLog) + tracer, tracerCloser := setupTracing(ctx, staticConfiguration.Tracing) observabilityMgr := middleware.NewObservabilityMgr(*staticConfiguration, metricsRegistry, semConvMetricRegistry, accessLog, tracer, tracerCloser) // Entrypoints @@ -586,12 +587,12 @@ func appendCertMetric(gauge gokitmetrics.Gauge, certificate *x509.Certificate) { gauge.With(labels...).Set(notAfter) } -func setupAccessLog(conf *types.AccessLog) *accesslog.Handler { +func setupAccessLog(ctx context.Context, conf *types.AccessLog) *accesslog.Handler { if conf == nil { return nil } - accessLoggerMiddleware, err := accesslog.NewHandler(conf) + accessLoggerMiddleware, err := accesslog.NewHandler(ctx, conf) if err != nil { log.Warn().Err(err).Msg("Unable to create access logger") return nil @@ -600,12 +601,12 @@ func setupAccessLog(conf *types.AccessLog) *accesslog.Handler { return accessLoggerMiddleware } -func setupTracing(conf *static.Tracing) (*tracing.Tracer, io.Closer) { +func setupTracing(ctx context.Context, conf *static.Tracing) (*tracing.Tracer, io.Closer) { if conf == nil { return nil, nil } - tracer, closer, err := tracing.NewTracing(conf) + tracer, closer, err := tracing.NewTracing(ctx, conf) if err != nil { log.Warn().Err(err).Msg("Unable to create tracer") return nil, nil diff --git a/docs/content/migration/v3.md b/docs/content/migration/v3.md index e3cbe8a9b..7b676ee9e 100644 --- a/docs/content/migration/v3.md +++ b/docs/content/migration/v3.md @@ -322,9 +322,11 @@ and Traefik now keeps them encoded to avoid any ambiguity. ## v3.5.0 -### TraceVerbosity on Routers and Entrypoints +### Observability -Starting with v3.5, a new `traceVerbosity` option is available for both entrypoints and routers. +#### TraceVerbosity on Routers and Entrypoints + +Starting with `v3.5.0`, a new `traceVerbosity` option is available for both entrypoints and routers. This option allows you to control the level of detail for tracing spans. Routers can override the value inherited from their entrypoint. @@ -339,3 +341,20 @@ Possible values are: - `detailed`: enables the creation of additional spans for each middleware executed for each request processed by a router. See the updated documentation for [entrypoints](../reference/install-configuration/entrypoints.md) and [dynamic routers](../reference/dynamic-configuration/file.md#observability-options). + +#### K8s Resource Attributes + +Since `v3.5.0`, the semconv attributes `k8s.pod.name` and `k8s.pod.uid` are injected automatically in OTel resource attributes when OTel tracing/logs/metrics are enabled. + +For that purpose, the following right has to be added to the Traefik Kubernetes RBACs: + +```yaml + ... + - apiGroups: + - "" + resources: + - pods + verbs: + - get + ... +``` diff --git a/docs/content/reference/dynamic-configuration/kubernetes-crd-rbac.yml b/docs/content/reference/dynamic-configuration/kubernetes-crd-rbac.yml index 87a438386..eb3f708bf 100644 --- a/docs/content/reference/dynamic-configuration/kubernetes-crd-rbac.yml +++ b/docs/content/reference/dynamic-configuration/kubernetes-crd-rbac.yml @@ -15,6 +15,14 @@ rules: - get - list - watch + # The pods right is needed to inject k8s.pod.uid and k8s.pod.name OTel attributes. + # When OTel tracing/logs/metrics are not enabled, this rule is not needed. + - apiGroups: + - "" + resources: + - pods + verbs: + - get - apiGroups: - discovery.k8s.io resources: diff --git a/docs/content/reference/dynamic-configuration/kubernetes-gateway-rbac.yml b/docs/content/reference/dynamic-configuration/kubernetes-gateway-rbac.yml index 31b0837c8..c03dc0147 100644 --- a/docs/content/reference/dynamic-configuration/kubernetes-gateway-rbac.yml +++ b/docs/content/reference/dynamic-configuration/kubernetes-gateway-rbac.yml @@ -11,6 +11,14 @@ rules: verbs: - list - watch + # The pods get right is needed to inject k8s.pod.uid and k8s.pod.name in OTel attributes. + # When OTel tracing/logs/metrics are not enabled, this rule is not needed. + - apiGroups: + - "" + resources: + - pods + verbs: + - get - apiGroups: - "" resources: diff --git a/pkg/logs/otel.go b/pkg/logs/otel.go index bc8f95443..f1e1711d5 100644 --- a/pkg/logs/otel.go +++ b/pkg/logs/otel.go @@ -1,6 +1,7 @@ package logs import ( + "context" "encoding/json" "fmt" "reflect" @@ -12,12 +13,12 @@ import ( ) // SetupOTelLogger sets up the OpenTelemetry logger. -func SetupOTelLogger(logger zerolog.Logger, config *types.OTelLog) (zerolog.Logger, error) { +func SetupOTelLogger(ctx context.Context, logger zerolog.Logger, config *types.OTelLog) (zerolog.Logger, error) { if config == nil { return logger, nil } - provider, err := config.NewLoggerProvider() + provider, err := config.NewLoggerProvider(ctx) if err != nil { return zerolog.Logger{}, fmt.Errorf("setting up OpenTelemetry logger provider: %w", err) } diff --git a/pkg/logs/otel_test.go b/pkg/logs/otel_test.go index c55137cf1..75f73be33 100644 --- a/pkg/logs/otel_test.go +++ b/pkg/logs/otel_test.go @@ -171,7 +171,7 @@ func TestLog(t *testing.T) { out := zerolog.MultiLevelWriter(zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.RFC3339}) logger := zerolog.New(out).With().Caller().Logger() - logger, err := SetupOTelLogger(logger, config) + logger, err := SetupOTelLogger(t.Context(), logger, config) require.NoError(t, err) ctx := trace.ContextWithSpanContext(t.Context(), trace.NewSpanContext(trace.SpanContextConfig{ diff --git a/pkg/metrics/otel.go b/pkg/metrics/otel.go index b29a89f29..5da580bd8 100644 --- a/pkg/metrics/otel.go +++ b/pkg/metrics/otel.go @@ -217,6 +217,7 @@ func newOpenTelemetryMeterProvider(ctx context.Context, config *types.OTLP) (*sd resource.WithOS(), resource.WithProcess(), resource.WithTelemetrySDK(), + resource.WithDetectors(types.K8sAttributesDetector{}), // The following order allows the user to override the service name and version, // as well as any other attributes set by the above detectors. resource.WithAttributes( diff --git a/pkg/middlewares/accesslog/logger.go b/pkg/middlewares/accesslog/logger.go index 246556195..c9a3242df 100644 --- a/pkg/middlewares/accesslog/logger.go +++ b/pkg/middlewares/accesslog/logger.go @@ -85,7 +85,7 @@ func (h *Handler) AliceConstructor() alice.Constructor { } // NewHandler creates a new Handler. -func NewHandler(config *types.AccessLog) (*Handler, error) { +func NewHandler(ctx context.Context, config *types.AccessLog) (*Handler, error) { var file io.WriteCloser = noopCloser{os.Stdout} if len(config.FilePath) > 0 { f, err := openAccessLogFile(config.FilePath) @@ -116,7 +116,7 @@ func NewHandler(config *types.AccessLog) (*Handler, error) { } if config.OTLP != nil { - otelLoggerProvider, err := config.OTLP.NewLoggerProvider() + otelLoggerProvider, err := config.OTLP.NewLoggerProvider(ctx) if err != nil { return nil, fmt.Errorf("setting up OpenTelemetry logger provider: %w", err) } diff --git a/pkg/middlewares/accesslog/logger_test.go b/pkg/middlewares/accesslog/logger_test.go index 7babf8121..fbf9fd380 100644 --- a/pkg/middlewares/accesslog/logger_test.go +++ b/pkg/middlewares/accesslog/logger_test.go @@ -85,7 +85,7 @@ func TestOTelAccessLog(t *testing.T) { }, }, } - logHandler, err := NewHandler(config) + logHandler, err := NewHandler(t.Context(), config) require.NoError(t, err) t.Cleanup(func() { err := logHandler.Close() @@ -138,7 +138,7 @@ func TestLogRotation(t *testing.T) { rotatedFileName := fileName + ".rotated" config := &types.AccessLog{FilePath: fileName, Format: CommonFormat} - logHandler, err := NewHandler(config) + logHandler, err := NewHandler(t.Context(), config) require.NoError(t, err) t.Cleanup(func() { err := logHandler.Close() @@ -282,7 +282,7 @@ func TestLoggerHeaderFields(t *testing.T) { Fields: &test.accessLogFields, } - logger, err := NewHandler(config) + logger, err := NewHandler(t.Context(), config) require.NoError(t, err) t.Cleanup(func() { err := logger.Close() @@ -979,7 +979,7 @@ func captureStdout(t *testing.T) (out *os.File, restoreStdout func()) { func doLoggingTLSOpt(t *testing.T, config *types.AccessLog, enableTLS, tracing bool) { t.Helper() - logger, err := NewHandler(config) + logger, err := NewHandler(t.Context(), config) require.NoError(t, err) t.Cleanup(func() { err := logger.Close() @@ -1076,7 +1076,7 @@ func logWriterTestHandlerFunc(rw http.ResponseWriter, r *http.Request) { func doLoggingWithAbortedStream(t *testing.T, config *types.AccessLog) { t.Helper() - logger, err := NewHandler(config) + logger, err := NewHandler(t.Context(), config) require.NoError(t, err) t.Cleanup(func() { err := logger.Close() diff --git a/pkg/tracing/tracing.go b/pkg/tracing/tracing.go index 76a3d3302..c2f998eae 100644 --- a/pkg/tracing/tracing.go +++ b/pkg/tracing/tracing.go @@ -25,11 +25,11 @@ import ( // Backend is an abstraction for tracking backend (OpenTelemetry, ...). type Backend interface { - Setup(serviceName string, sampleRate float64, resourceAttributes map[string]string) (trace.Tracer, io.Closer, error) + Setup(ctx context.Context, serviceName string, sampleRate float64, resourceAttributes map[string]string) (trace.Tracer, io.Closer, error) } // NewTracing Creates a Tracing. -func NewTracing(conf *static.Tracing) (*Tracer, io.Closer, error) { +func NewTracing(ctx context.Context, conf *static.Tracing) (*Tracer, io.Closer, error) { var backend Backend if conf.OTLP != nil { @@ -44,7 +44,7 @@ func NewTracing(conf *static.Tracing) (*Tracer, io.Closer, error) { otel.SetTextMapPropagator(autoprop.NewTextMapPropagator()) - tr, closer, err := backend.Setup(conf.ServiceName, conf.SampleRate, conf.ResourceAttributes) + tr, closer, err := backend.Setup(ctx, conf.ServiceName, conf.SampleRate, conf.ResourceAttributes) if err != nil { return nil, nil, err } @@ -84,13 +84,6 @@ func InjectContextIntoCarrier(req *http.Request) { propagator.Inject(req.Context(), propagation.HeaderCarrier(req.Header)) } -// SetStatusErrorf flags the span as in error and log an event. -func SetStatusErrorf(ctx context.Context, format string, args ...interface{}) { - if span := trace.SpanFromContext(ctx); span != nil { - span.SetStatus(codes.Error, fmt.Sprintf(format, args...)) - } -} - // Span is trace.Span wrapping the Traefik TracerProvider. type Span struct { trace.Span diff --git a/pkg/tracing/tracing_test.go b/pkg/tracing/tracing_test.go index 9507b528c..e3cc39cea 100644 --- a/pkg/tracing/tracing_test.go +++ b/pkg/tracing/tracing_test.go @@ -350,7 +350,7 @@ func TestTracing(t *testing.T) { }, } - tracer, closer, err := NewTracing(tracingConfig) + tracer, closer, err := NewTracing(t.Context(), tracingConfig) require.NoError(t, err) t.Cleanup(func() { _ = closer.Close() @@ -402,7 +402,7 @@ func TestTracerProvider(t *testing.T) { otlpConfig.SetDefaults() config := &static.Tracing{OTLP: otlpConfig} - tracer, closer, err := NewTracing(config) + tracer, closer, err := NewTracing(t.Context(), config) if err != nil { t.Fatal(err) } diff --git a/pkg/types/k8sdetector.go b/pkg/types/k8sdetector.go new file mode 100644 index 000000000..d87589c1f --- /dev/null +++ b/pkg/types/k8sdetector.go @@ -0,0 +1,70 @@ +package types + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + + "github.com/rs/zerolog/log" + "go.opentelemetry.io/otel/sdk/resource" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" + kerror "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kclientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +// K8sAttributesDetector detects the metadata of the Traefik pod running in a Kubernetes cluster. +// It reads the pod name from the hostname file and the namespace from the service account namespace file and queries the Kubernetes API to get the pod's UID. +type K8sAttributesDetector struct{} + +func (K8sAttributesDetector) Detect(ctx context.Context) (*resource.Resource, error) { + attrs := os.Getenv("OTEL_RESOURCE_ATTRIBUTES") + if strings.Contains(attrs, string(semconv.K8SPodNameKey)) || strings.Contains(attrs, string(semconv.K8SPodUIDKey)) { + return resource.Empty(), nil + } + + // The InClusterConfig function returns a config for the Kubernetes API server + // when it is running inside a Kubernetes cluster. + config, err := rest.InClusterConfig() + if err != nil && errors.Is(err, rest.ErrNotInCluster) { + return resource.Empty(), nil + } + if err != nil { + return nil, fmt.Errorf("creating in cluster config: %w", err) + } + + client, err := kclientset.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("creating Kubernetes client: %w", err) + } + + podName, err := os.Hostname() + if err != nil { + return nil, fmt.Errorf("getting pod name: %w", err) + } + + podNamespaceBytes, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace") + if err != nil { + return nil, fmt.Errorf("getting pod namespace: %w", err) + } + podNamespace := string(podNamespaceBytes) + + pod, err := client.CoreV1().Pods(podNamespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil && kerror.IsForbidden(err) { + log.Error().Err(err).Msg("Unable to build K8s resource attributes for Traefik pod") + return resource.Empty(), nil + } + if err != nil { + return nil, fmt.Errorf("getting pod metadata: %w", err) + } + + // To avoid version conflicts with other detectors, we use a Schemaless resource. + return resource.NewSchemaless( + semconv.K8SPodUID(string(pod.UID)), + semconv.K8SPodName(pod.Name), + semconv.K8SNamespaceName(podNamespace), + ), nil +} diff --git a/pkg/types/logs.go b/pkg/types/logs.go index a1af7f44c..c476d166d 100644 --- a/pkg/types/logs.go +++ b/pkg/types/logs.go @@ -13,7 +13,7 @@ import ( "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp" otelsdk "go.opentelemetry.io/otel/sdk/log" "go.opentelemetry.io/otel/sdk/resource" - semconv "go.opentelemetry.io/otel/semconv/v1.27.0" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" "google.golang.org/grpc/credentials" "google.golang.org/grpc/encoding/gzip" ) @@ -164,7 +164,7 @@ func (o *OTelLog) SetDefaults() { } // NewLoggerProvider creates a new OpenTelemetry logger provider. -func (o *OTelLog) NewLoggerProvider() (*otelsdk.LoggerProvider, error) { +func (o *OTelLog) NewLoggerProvider(ctx context.Context) (*otelsdk.LoggerProvider, error) { var ( err error exporter otelsdk.Exporter @@ -178,23 +178,27 @@ func (o *OTelLog) NewLoggerProvider() (*otelsdk.LoggerProvider, error) { return nil, fmt.Errorf("setting up exporter: %w", err) } - attr := []attribute.KeyValue{ - semconv.ServiceNameKey.String(o.ServiceName), - semconv.ServiceVersionKey.String(version.Version), - } - + var resAttrs []attribute.KeyValue for k, v := range o.ResourceAttributes { - attr = append(attr, attribute.String(k, v)) + resAttrs = append(resAttrs, attribute.String(k, v)) } - res, err := resource.New(context.Background(), - resource.WithAttributes(attr...), + res, err := resource.New(ctx, resource.WithContainer(), - resource.WithFromEnv(), resource.WithHost(), resource.WithOS(), resource.WithProcess(), resource.WithTelemetrySDK(), + resource.WithDetectors(K8sAttributesDetector{}), + // The following order allows the user to override the service name and version, + // as well as any other attributes set by the above detectors. + resource.WithAttributes( + semconv.ServiceName(o.ServiceName), + semconv.ServiceVersion(version.Version), + ), + resource.WithAttributes(resAttrs...), + // Use the environment variables to allow overriding above resource attributes. + resource.WithFromEnv(), ) if err != nil { return nil, fmt.Errorf("building resource: %w", err) diff --git a/pkg/types/tracing.go b/pkg/types/tracing.go index 0ee66ea3c..d07bb43c5 100644 --- a/pkg/types/tracing.go +++ b/pkg/types/tracing.go @@ -17,7 +17,7 @@ import ( "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" "go.opentelemetry.io/otel/sdk/resource" sdktrace "go.opentelemetry.io/otel/sdk/trace" - semconv "go.opentelemetry.io/otel/semconv/v1.27.0" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" "go.opentelemetry.io/otel/trace" "google.golang.org/grpc/credentials" "google.golang.org/grpc/encoding/gzip" @@ -52,7 +52,7 @@ func (c *OTelTracing) SetDefaults() { } // Setup sets up the tracer. -func (c *OTelTracing) Setup(serviceName string, sampleRate float64, resourceAttributes map[string]string) (trace.Tracer, io.Closer, error) { +func (c *OTelTracing) Setup(ctx context.Context, serviceName string, sampleRate float64, resourceAttributes map[string]string) (trace.Tracer, io.Closer, error) { var ( err error exporter *otlptrace.Exporter @@ -66,23 +66,27 @@ func (c *OTelTracing) Setup(serviceName string, sampleRate float64, resourceAttr return nil, nil, fmt.Errorf("setting up exporter: %w", err) } - attr := []attribute.KeyValue{ - semconv.ServiceNameKey.String(serviceName), - semconv.ServiceVersionKey.String(version.Version), - } - + var resAttrs []attribute.KeyValue for k, v := range resourceAttributes { - attr = append(attr, attribute.String(k, v)) + resAttrs = append(resAttrs, attribute.String(k, v)) } - res, err := resource.New(context.Background(), - resource.WithAttributes(attr...), + res, err := resource.New(ctx, resource.WithContainer(), - resource.WithFromEnv(), resource.WithHost(), resource.WithOS(), resource.WithProcess(), resource.WithTelemetrySDK(), + resource.WithDetectors(K8sAttributesDetector{}), + // The following order allows the user to override the service name and version, + // as well as any other attributes set by the above detectors. + resource.WithAttributes( + semconv.ServiceName(serviceName), + semconv.ServiceVersion(version.Version), + ), + resource.WithAttributes(resAttrs...), + // Use the environment variables to allow overriding above resource attributes. + resource.WithFromEnv(), ) if err != nil { return nil, nil, fmt.Errorf("building resource: %w", err)