From 3e9eaa293a6c6cf7491634754efec7ea5a637527 Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kristoffer@tailscale.com>
Date: Wed, 14 Aug 2024 16:45:40 +0200
Subject: [PATCH] cmd/tailscaled: count all health messages

Signed-off-by: Kristoffer Dalby <kristoffer@tailscale.com>
---
 cmd/tailscaled/tailscaled.go | 18 ++++++++++++++++++
 control/controlclient/map.go | 15 ---------------
 health/health.go             | 12 ++++++++++++
 3 files changed, 30 insertions(+), 15 deletions(-)

diff --git a/cmd/tailscaled/tailscaled.go b/cmd/tailscaled/tailscaled.go
index 6dbf6c982..dea1d78bb 100644
--- a/cmd/tailscaled/tailscaled.go
+++ b/cmd/tailscaled/tailscaled.go
@@ -63,6 +63,7 @@ import (
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/multierr"
 	"tailscale.com/util/osshare"
+	"tailscale.com/util/usermetric"
 	"tailscale.com/version"
 	"tailscale.com/version/distro"
 	"tailscale.com/wgengine"
@@ -340,6 +341,13 @@ func run() (err error) {
 
 	sys := new(tsd.System)
 
+	health := sys.HealthTracker()
+	metricHealthMessages.Set(healthMessageLabel{
+		Severity: "warning",
+	}, expvar.Func(func() any {
+		return health.OverallErrorCount()
+	}))
+
 	// Parse config, if specified, to fail early if it's invalid.
 	var conf *conffile.Config
 	if args.confFile != "" {
@@ -919,3 +927,13 @@ func applyIntegrationTestEnvKnob() {
 		}
 	}
 }
+
+type healthMessageLabel struct {
+	Severity string
+}
+
+var metricHealthMessages = usermetric.NewMultiLabelMap[healthMessageLabel](
+	"tailscaled_health_messages",
+	"gauge",
+	"A gauge of health messages from control, by severity",
+)
diff --git a/control/controlclient/map.go b/control/controlclient/map.go
index 19590db7b..436808995 100644
--- a/control/controlclient/map.go
+++ b/control/controlclient/map.go
@@ -7,7 +7,6 @@ import (
 	"cmp"
 	"context"
 	"encoding/json"
-	"expvar"
 	"fmt"
 	"maps"
 	"net"
@@ -33,7 +32,6 @@ import (
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/mak"
 	"tailscale.com/util/set"
-	"tailscale.com/util/usermetric"
 	"tailscale.com/wgengine/filter"
 )
 
@@ -346,9 +344,6 @@ func (ms *mapSession) updateStateFromResponse(resp *tailcfg.MapResponse) {
 	}
 	if resp.Health != nil {
 		ms.lastHealth = resp.Health
-		warnings := expvar.Int{}
-		warnings.Set(int64(len(resp.Health)))
-		metricHealthMessages.Set(healthMessageLabel{Severity: "warning"}, &warnings)
 	}
 	if resp.TKAInfo != nil {
 		ms.lastTKAInfo = resp.TKAInfo
@@ -358,16 +353,6 @@ func (ms *mapSession) updateStateFromResponse(resp *tailcfg.MapResponse) {
 	}
 }
 
-type healthMessageLabel struct {
-	Severity string
-}
-
-var metricHealthMessages = usermetric.NewMultiLabelMap[healthMessageLabel](
-	"tailscaled_health_messages",
-	"gauge",
-	"A gauge of health messages from control, by severity",
-)
-
 var (
 	patchDERPRegion   = clientmetric.NewCounter("controlclient_patch_derp")
 	patchEndpoints    = clientmetric.NewCounter("controlclient_patch_endpoints")
diff --git a/health/health.go b/health/health.go
index 10549b523..f7e220e70 100644
--- a/health/health.go
+++ b/health/health.go
@@ -897,6 +897,18 @@ func (t *Tracker) OverallError() error {
 	return t.multiErrLocked()
 }
 
+// OverallErrorCount returns the number of errors currently known to the
+// Tracker.
+func (t *Tracker) OverallErrorCount() int64 {
+	if t.nil() {
+		return 0
+	}
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	t.updateBuiltinWarnablesLocked()
+	return int64(len(t.stringsLocked()))
+}
+
 // Strings() returns a string array containing the Text of all Warnings
 // currently known to the Tracker. These strings can be presented to the
 // user, although ideally you would use the Code property on each Warning