k8s-operator: send operator logs to tailscale

This commit modifies the k8s operator to wrap its logger using the logtail
logger provided via the tsnet server. This causes any logs written by
the operator to make their way to Tailscale in the same fashion as
wireguard logs to be used by support.

Operator logs can be determined by the "k8s:" prefix within the log line.
This functionality can also be opted-out of entirely using the
"TS_NO_LOGS_NO_SUPPORT" environment variable.

This is implemented by reusing kzap's core with a different sink that
just writes to the logger.Logf function provided.

Fixes https://github.com/tailscale/corp/issues/32037

Signed-off-by: David Bond <davidsbond93@gmail.com>

cmd/k8s-operator/logger.go

@@ -0,0 +1,41 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+	kzap "sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"tailscale.com/types/logger"
+)
+
+type (
+	logfSink struct {
+		logf logger.Logf
+	}
+)
+
+// wrapZapCore returns a zapcore.Core implementation that splits the core chain using zapcore.NewTee. This causes
+// logs to be simultaneously written to both the original core and the provided logger.Logf function.
+func wrapZapCore(core zapcore.Core, logf logger.Logf) zapcore.Core {
+	// We use a tee logger here so that logs are written to stdout/stderr normally while at the same time being
+	// sent upstream.
+	return zapcore.NewTee(core, zapcore.NewCore(&kzap.KubeAwareEncoder{
+		Encoder: zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()),
+		Verbose: true,
+	}, &logfSink{logf: logf}, zap.LevelEnablerFunc(func(lvl zapcore.Level) bool {
+		return true
+	})))
+}
+
+func (l *logfSink) Write(p []byte) (n int, err error) {
+	l.logf("k8s: %s", p)
+	return len(p), nil
+}
+
+func (l *logfSink) Sync() error {
+	return nil
+}

cmd/k8s-operator/operator.go

@@ -44,6 +44,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"tailscale.com/envknob"
 
 	"tailscale.com/client/local"
 	"tailscale.com/client/tailscale"
@@ -133,6 +134,14 @@ func main() {
 			}
 		}()
 	}
+
+	// Operator log uploads can be opted-out using the "TS_NO_LOGS_NO_SUPPORT" environment variable.
+	if !envknob.NoLogsNoSupport() {
+		zlog = zlog.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core {
+			return wrapZapCore(core, s.Logtailf())
+		}))
+	}
+
 	rOpts := reconcilerOpts{
 		log:                           zlog,
 		tsServer:                      s,

cmd/k8s-operator/sts.go

@@ -439,12 +439,12 @@ func (a *tailscaleSTSReconciler) provisionSecrets(ctx context.Context, logger *z
 		}
 
 		if orig != nil && !apiequality.Semantic.DeepEqual(latest, orig) {
-			logger.Debugf("patching the existing proxy Secret with tailscaled config %s", sanitizeConfigBytes(latestConfig))
+			logger.With("config", sanitizeConfig(latestConfig)).Debugf("patching the existing proxy Secret")
 			if err = a.Patch(ctx, secret, client.MergeFrom(orig)); err != nil {
 				return nil, err
 			}
 		} else {
-			logger.Debugf("creating a new Secret for the proxy with tailscaled config %s", sanitizeConfigBytes(latestConfig))
+			logger.With("config", sanitizeConfig(latestConfig)).Debugf("creating a new Secret for the proxy")
 			if err = a.Create(ctx, secret); err != nil {
 				return nil, err
 			}
@@ -494,17 +494,16 @@ func (a *tailscaleSTSReconciler) provisionSecrets(ctx context.Context, logger *z
 	return secretNames, nil
 }
 
-// sanitizeConfigBytes returns ipn.ConfigVAlpha in string form with redacted
-// auth key.
-func sanitizeConfigBytes(c ipn.ConfigVAlpha) string {
+// sanitizeConfig returns an ipn.ConfigVAlpha with sensitive fields redacted. Since we pump everything
+// into JSON-encoded logs it's easier to read this with a .With method than converting it to a string.
+func sanitizeConfig(c ipn.ConfigVAlpha) ipn.ConfigVAlpha {
+	// Explicitly set AuthKey to nil because we never want it appearing in logs. Never populate this with the
+	// actual auth key.
 	if c.AuthKey != nil {
 		c.AuthKey = ptr.To("**redacted**")
 	}
-	sanitizedBytes, err := json.Marshal(c)
-	if err != nil {
-		return "invalid config"
-	}
-	return string(sanitizedBytes)
+
+	return c
 }
 
 // DeviceInfo returns the device ID, hostname, IPs and capver for the Tailscale device that acts as an operator proxy.