tsd, all: add Sys.ExtraRootCAs, plumb through TLS dial paths

Add ExtraRootCAs *x509.CertPool to tsd.System and plumb it through
the control client, noise transport, DERP, and wgengine layers so
that platforms like Android can inject user-installed CA certificates
into Go's TLS verification.

tlsdial.Config now honors base.RootCAs as additional trusted roots,
tried after system roots and before the baked-in LetsEncrypt fallback.
SetConfigExpectedCert gets the same treatment for domain-fronted DERP.

The Android client will set sys.ExtraRootCAs with a pool built from
x509.SystemCertPool + user-installed certs obtained via the Android
KeyStore API, replacing the current SSL_CERT_DIR environment variable
approach.

Updates #8085

Change-Id: Iecce0fd140cd5aa0331b124e55a7045e24d8e0c2
Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

cmd/tailscaled/tailscaled.go

@@ -744,6 +744,7 @@ func tryEngine(logf logger.Logf, sys *tsd.System, name string) (onlyNetstack boo
 		ListenPort:    args.port,
 		NetMon:        sys.NetMon.Get(),
 		HealthTracker: sys.HealthTracker.Get(),
+		ExtraRootCAs:  sys.ExtraRootCAs,
 		Metrics:       sys.UserMetricsRegistry(),
 		Dialer:        sys.Dialer.Get(),
 		SetSubsystem:  sys.Set,

cmd/tsconnect/wasm/wasm_js.go

@@ -110,6 +110,7 @@ func newIPN(jsConfig js.Value) map[string]any {
 		SetSubsystem:  sys.Set,
 		ControlKnobs:  sys.ControlKnobs(),
 		HealthTracker: sys.HealthTracker.Get(),
+		ExtraRootCAs:  sys.ExtraRootCAs,
 		Metrics:       sys.UserMetricsRegistry(),
 		EventBus:      sys.Bus.Get(),
 	})

control/controlclient/direct.go

@@ -9,6 +9,8 @@ import (
 	"context"
 	"crypto"
 	"crypto/sha256"
+	"crypto/tls"
+	"crypto/x509"
 	"encoding/binary"
 	"encoding/json"
 	"errors"
@@ -74,6 +76,7 @@ type Direct struct {
 	logf              logger.Logf
 	netMon            *netmon.Monitor // non-nil
 	health            *health.Tracker
+	extraRootCAs      *x509.CertPool // additional trusted root CAs; or nil
 	busClient         *eventbus.Client
 	clientVersionPub  *eventbus.Publisher[tailcfg.ClientVersion]
 	autoUpdatePub     *eventbus.Publisher[AutoUpdate]
@@ -141,6 +144,7 @@ type Options struct {
 	NoiseTestClient      *http.Client // optional HTTP client to use for noise RPCs (tests only)
 	DebugFlags           []string     // debug settings to send to control
 	HealthTracker        *health.Tracker
+	ExtraRootCAs         *x509.CertPool      // additional trusted root CAs; or nil
 	PopBrowserURL        func(url string)    // optional func to open browser
 	Dialer               *tsdial.Dialer      // non-nil
 	C2NHandler           http.Handler        // or nil
@@ -297,6 +301,12 @@ func NewDirect(opts Options) (*Direct, error) {
 				f(tr)
 			}
 		}
+		if opts.ExtraRootCAs != nil {
+			if tr.TLSClientConfig == nil {
+				tr.TLSClientConfig = &tls.Config{}
+			}
+			tr.TLSClientConfig.RootCAs = opts.ExtraRootCAs
+		}
 		tr.TLSClientConfig = tlsdial.Config(opts.HealthTracker, tr.TLSClientConfig)
 		var dialFunc netx.DialFunc
 		dialFunc, interceptedDial = makeScreenTimeDetectingDialFunc(opts.Dialer.SystemDial)
@@ -324,6 +334,7 @@ func NewDirect(opts Options) (*Direct, error) {
 		debugFlags:        opts.DebugFlags,
 		netMon:            netMon,
 		health:            opts.HealthTracker,
+		extraRootCAs:      opts.ExtraRootCAs,
 		pinger:            opts.Pinger,
 		polc:              cmp.Or(opts.PolicyClient, policyclient.Client(policyclient.NoPolicyClient{})),
 		popBrowser:        opts.PopBrowserURL,
@@ -1631,6 +1642,7 @@ func (c *Direct) getNoiseClient() (*ts2021.Client, error) {
 			Logf:          c.logf,
 			NetMon:        c.netMon,
 			HealthTracker: c.health,
+			ExtraRootCAs:  c.extraRootCAs,
 			DialPlan:      dp,
 		})
 		if err != nil {

control/controlhttp/client.go

@@ -479,6 +479,9 @@ func (a *Dialer) tryURLUpgrade(ctx context.Context, u *url.URL, optAddr netip.Ad
 	// Disable HTTP2, since h2 can't do protocol switching.
 	tr.TLSClientConfig.NextProtos = []string{}
 	tr.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{}
+	if a.ExtraRootCAs != nil {
+		tr.TLSClientConfig.RootCAs = a.ExtraRootCAs
+	}
 	tr.TLSClientConfig = tlsdial.Config(a.HealthTracker, tr.TLSClientConfig)
 	if !tr.TLSClientConfig.InsecureSkipVerify {
 		panic("unexpected") // should be set by tlsdial.Config

control/controlhttp/constants.go

@@ -4,6 +4,7 @@
 package controlhttp
 
 import (
+	"crypto/x509"
 	"net/http"
 	"net/url"
 	"sync/atomic"
@@ -85,6 +86,9 @@ type Dialer struct {
 	// HealthTracker, if non-nil, is the health tracker to use.
 	HealthTracker *health.Tracker
 
+	// ExtraRootCAs, if non-nil, specifies additional trusted root CAs for TLS.
+	ExtraRootCAs *x509.CertPool
+
 	// DialPlan, if set, contains instructions from the control server on
 	// how to connect to it. If present, we will try the methods in this
 	// plan before falling back to DNS.

control/ts2021/client.go

@@ -7,6 +7,7 @@ import (
 	"bytes"
 	"cmp"
 	"context"
+	"crypto/x509"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -86,6 +87,9 @@ type ClientOpts struct {
 	// HealthTracker, if non-nil, is the health tracker to use.
 	HealthTracker *health.Tracker
 
+	// ExtraRootCAs, if non-nil, specifies additional trusted root CAs for TLS.
+	ExtraRootCAs *x509.CertPool
+
 	// DialPlan, if set, is a function that should return an explicit plan
 	// on how to connect to the server.
 	DialPlan func() *tailcfg.ControlDialPlan
@@ -252,6 +256,7 @@ func (nc *Client) dial(ctx context.Context) (*Conn, error) {
 		Logf:            nc.logf,
 		NetMon:          nc.opts.NetMon,
 		HealthTracker:   nc.opts.HealthTracker,
+		ExtraRootCAs:    nc.opts.ExtraRootCAs,
 		Clock:           tstime.StdClock{},
 	}
 	clientConn, err := chd.Dial(ctx)

ipn/ipnlocal/local.go

@@ -2662,6 +2662,7 @@ func (b *LocalBackend) startLocked(opts ipn.Options) error {
 		DiscoPublicKey:       discoPublic,
 		DebugFlags:           b.controlDebugFlags(),
 		HealthTracker:        b.health,
+		ExtraRootCAs:         b.sys.ExtraRootCAs,
 		PolicyClient:         b.sys.PolicyClientOrDefault(),
 		Pinger:               b,
 		PopBrowserURL:        b.tellClientToBrowseToURL,

net/tlsdial/tlsdial.go

@@ -59,15 +59,26 @@ var mitmBlockWarnable = health.Register(&health.Warnable{
 // the baked-in LetsEncrypt roots as a fallback validation method.
 //
 // If base is non-nil, it's cloned as the base config before
-// being configured and returned.
+// being configured and returned. If base.RootCAs is non-nil, it is
+// used as an additional set of trusted roots (after system roots,
+// before baked-in LetsEncrypt roots). This is used on Android to
+// trust user-installed CA certificates that Go's crypto/x509
+// does not see.
+//
 // If ht is non-nil, it's used to report health errors.
 func Config(ht *health.Tracker, base *tls.Config) *tls.Config {
+	var extraRoots *x509.CertPool
+	if base != nil {
+		extraRoots = base.RootCAs
+	}
+
 	var conf *tls.Config
 	if base == nil {
 		conf = new(tls.Config)
 	} else {
 		conf = base.Clone()
 	}
+	conf.RootCAs = nil // we do our own verification in VerifyConnection
 
 	// Note: we do NOT set conf.ServerName here (as we accidentally did
 	// previously), as this path is also used when dialing an HTTPS proxy server
@@ -165,7 +176,26 @@ func Config(ht *health.Tracker, base *tls.Config) *tls.Config {
 		if debug() {
 			log.Printf("tlsdial(sys %q): %v", dialedHost, errSys)
 		}
-		if !buildfeatures.HasBakedRoots || (errSys == nil && !debug()) {
+		if errSys == nil && !debug() {
+			return nil
+		}
+
+		// If extra roots were provided (e.g. user-installed CAs on
+		// Android), try those next.
+		if extraRoots != nil {
+			opts.Roots = extraRoots
+			_, errExtra := cs.PeerCertificates[0].Verify(opts)
+			if debug() {
+				log.Printf("tlsdial(extra %q): %v", dialedHost, errExtra)
+			}
+			if errExtra == nil {
+				atomic.AddInt32(&counterFallbackOK, 1)
+				return nil
+			}
+			opts.Roots = nil // reset for baked roots check
+		}
+
+		if !buildfeatures.HasBakedRoots {
 			return errSys
 		}
 
@@ -178,7 +208,11 @@ func Config(ht *health.Tracker, base *tls.Config) *tls.Config {
 		} else if bakedErr != nil {
 			if _, loaded := tlsdialWarningPrinted.LoadOrStore(dialedHost, true); !loaded {
 				if errSys != nil {
-					log.Printf("tlsdial: error: server cert for %q failed both system roots & Let's Encrypt root validation", dialedHost)
+					if extraRoots != nil {
+						log.Printf("tlsdial: error: server cert for %q failed system roots, extra roots & Let's Encrypt root validation", dialedHost)
+					} else {
+						log.Printf("tlsdial: error: server cert for %q failed both system roots & Let's Encrypt root validation", dialedHost)
+					}
 				}
 			}
 		}
@@ -213,6 +247,10 @@ func SetConfigExpectedCert(c *tls.Config, certDNSName string) {
 		c.ServerName = certDNSName
 		return
 	}
+
+	extraRoots := c.RootCAs
+	c.RootCAs = nil
+
 	// Set InsecureSkipVerify to prevent crypto/tls from doing its
 	// own cert verification, but do the same work that it'd do
 	// (but using certDNSName) in the VerifyPeerCertificate hook.
@@ -242,7 +280,21 @@ func SetConfigExpectedCert(c *tls.Config, certDNSName string) {
 		if debug() {
 			log.Printf("tlsdial(sys %q/%q): %v", c.ServerName, certDNSName, errSys)
 		}
-		if !buildfeatures.HasBakedRoots || errSys == nil {
+		if errSys == nil {
+			return nil
+		}
+		if extraRoots != nil {
+			opts.Roots = extraRoots
+			_, errExtra := certs[0].Verify(opts)
+			if debug() {
+				log.Printf("tlsdial(extra %q/%q): %v", c.ServerName, certDNSName, errExtra)
+			}
+			if errExtra == nil {
+				return nil
+			}
+			opts.Roots = nil
+		}
+		if !buildfeatures.HasBakedRoots {
 			return errSys
 		}
 		opts.Roots = bakedroots.Get()

tsd/tsd.go

@@ -18,6 +18,7 @@
 package tsd
 
 import (
+	"crypto/x509"
 	"fmt"
 	"reflect"
 
@@ -63,6 +64,12 @@ type System struct {
 	PolicyClient   SubSystem[policyclient.Client]
 	HealthTracker  SubSystem[*health.Tracker]
 
+	// ExtraRootCAs, if non-nil, specifies additional trusted root CAs
+	// beyond the system roots. On Android, this includes user-installed
+	// CA certificates that Go's crypto/x509 does not see.
+	// It is plumbed through to tlsdial.Config via tls.Config.RootCAs.
+	ExtraRootCAs *x509.CertPool
+
 	// InitialConfig is initial server config, if any.
 	// It is nil if the node is not in declarative mode.
 	// This value is never updated after startup.

tsnet/tsnet.go

@@ -710,6 +710,7 @@ func (s *Server) start() (reterr error) {
 		SetSubsystem:  sys.Set,
 		ControlKnobs:  sys.ControlKnobs(),
 		HealthTracker: sys.HealthTracker.Get(),
+		ExtraRootCAs:  sys.ExtraRootCAs,
 		Metrics:       sys.UserMetricsRegistry(),
 	})
 	if err != nil {

wgengine/magicsock/derp.go

@@ -6,6 +6,7 @@ package magicsock
 import (
 	"bufio"
 	"context"
+	"crypto/tls"
 	"fmt"
 	"maps"
 	"net"
@@ -392,6 +393,9 @@ func (c *Conn) derpWriteChanForRegion(regionID int, peer key.NodePublic) chan de
 		return derpMap.Regions[regionID]
 	})
 	dc.HealthTracker = c.health
+	if c.extraRootCAs != nil {
+		dc.TLSConfig = &tls.Config{RootCAs: c.extraRootCAs}
+	}
 
 	dc.SetCanAckPings(true)
 	dc.NotePreferred(c.myDerp == regionID)

wgengine/magicsock/magicsock.go

@@ -9,6 +9,7 @@ import (
 	"bufio"
 	"bytes"
 	"context"
+	"crypto/x509"
 	"encoding/binary"
 	"errors"
 	"fmt"
@@ -167,6 +168,7 @@ type Conn struct {
 	onDERPRecv             func(int, key.NodePublic, []byte) bool // or nil, see Options.OnDERPRecv
 	netMon                 *netmon.Monitor                        // must be non-nil
 	health                 *health.Tracker                        // or nil
+	extraRootCAs           *x509.CertPool                         // additional trusted root CAs; or nil
 	controlKnobs           *controlknobs.Knobs                    // or nil
 
 	// ================================================================
@@ -481,6 +483,10 @@ type Options struct {
 	// report errors and warnings to.
 	HealthTracker *health.Tracker
 
+	// ExtraRootCAs, if non-nil, specifies additional trusted root CAs
+	// for TLS connections to DERP servers.
+	ExtraRootCAs *x509.CertPool
+
 	// Metrics specifies the metrics registry to record metrics to.
 	Metrics *usermetric.Registry
 
@@ -686,6 +692,7 @@ func NewConn(opts Options) (*Conn, error) {
 
 	c.netMon = opts.NetMon
 	c.health = opts.HealthTracker
+	c.extraRootCAs = opts.ExtraRootCAs
 	c.getPeerByKey = opts.PeerByKeyFunc
 
 	if err := c.rebind(keepCurrentPort); err != nil {

wgengine/userspace.go

@@ -7,6 +7,7 @@ import (
 	"bufio"
 	"context"
 	crand "crypto/rand"
+	"crypto/x509"
 	"errors"
 	"fmt"
 	"io"
@@ -236,6 +237,10 @@ type Config struct {
 	// If nil, a new Dialer is created.
 	Dialer *tsdial.Dialer
 
+	// ExtraRootCAs, if non-nil, specifies additional trusted root CAs for TLS
+	// connections (e.g. DERP). Passed through to magicsock.
+	ExtraRootCAs *x509.CertPool
+
 	// ControlKnobs is the set of control plane-provied knobs
 	// to use.
 	// If nil, defaults are used.
@@ -450,6 +455,7 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error)
 		IdleFunc:       e.tundev.IdleDuration,
 		NetMon:         e.netMon,
 		HealthTracker:  e.health,
+		ExtraRootCAs:   conf.ExtraRootCAs,
 		Metrics:        conf.Metrics,
 		ControlKnobs:   conf.ControlKnobs,
 		PeerByKeyFunc:  e.PeerByKey,