tailscale/tsnet/tsnet.go

// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause

// Package tsnet provides Tailscale as a library.
package tsnet

import (
	"context"
	crand "crypto/rand"
	"crypto/tls"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"log"
	"math"
	"net"
	"net/http"
	"net/netip"
	"os"
	"path/filepath"
	"runtime"
	"slices"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/tailscale/wireguard-go/tun"
	"tailscale.com/client/local"
	"tailscale.com/control/controlclient"
	"tailscale.com/envknob"
	_ "tailscale.com/feature/c2n"
	_ "tailscale.com/feature/condregister/identityfederation"
	_ "tailscale.com/feature/condregister/oauthkey"
	_ "tailscale.com/feature/condregister/portmapper"
	_ "tailscale.com/feature/condregister/useproxy"
	"tailscale.com/health"
	"tailscale.com/hostinfo"
	"tailscale.com/internal/client/tailscale"
	"tailscale.com/ipn"
	"tailscale.com/ipn/ipnauth"
	"tailscale.com/ipn/ipnlocal"
	"tailscale.com/ipn/ipnstate"
	"tailscale.com/ipn/localapi"
	"tailscale.com/ipn/store"
	"tailscale.com/ipn/store/mem"
	"tailscale.com/logpolicy"
	"tailscale.com/logtail"
	"tailscale.com/logtail/filch"
	"tailscale.com/net/memnet"
	"tailscale.com/net/netmon"
	"tailscale.com/net/proxymux"
	"tailscale.com/net/socks5"
	"tailscale.com/net/tsdial"
	"tailscale.com/tailcfg"
	"tailscale.com/tsd"
	"tailscale.com/types/bools"
	"tailscale.com/types/logger"
	"tailscale.com/types/logid"
	"tailscale.com/types/nettype"
	"tailscale.com/util/clientmetric"
	"tailscale.com/util/mak"
	"tailscale.com/util/set"
	"tailscale.com/util/testenv"
	"tailscale.com/wgengine"
	"tailscale.com/wgengine/netstack"
)

// Server is an embedded Tailscale server.
//
// Its exported fields may be changed until the first method call.
type Server struct {
	// Dir specifies the name of the directory to use for
	// state. If empty, a directory is selected automatically
	// under os.UserConfigDir (https://golang.org/pkg/os/#UserConfigDir).
	// based on the name of the binary.
	//
	// If you want to use multiple tsnet services in the same
	// binary, you will need to make sure that Dir is set uniquely
	// for each service. A good pattern for this is to have a
	// "base" directory (such as your mutable storage folder) and
	// then append the hostname on the end of it.
	Dir string

	// Store specifies the state store to use.
	//
	// If nil, a new FileStore is initialized at `Dir/tailscaled.state`.
	// See tailscale.com/ipn/store for supported stores.
	//
	// Logs will automatically be uploaded to log.tailscale.com,
	// where the configuration file for logging will be saved at
	// `Dir/tailscaled.log.conf`.
	Store ipn.StateStore

	// Hostname is the hostname to present to the control server.
	// If empty, the binary name is used.
	Hostname string

	// UserLogf, if non-nil, specifies the logger to use for logs generated by
	// the Server itself intended to be seen by the user such as the AuthURL for
	// login and status updates. If unset, log.Printf is used.
	UserLogf logger.Logf

	// Logf, if set is used for logs generated by the backend such as the
	// LocalBackend and MagicSock. It is verbose and intended for debugging.
	// If unset, logs are discarded.
	Logf logger.Logf

	// Ephemeral, if true, specifies that the instance should register
	// as an Ephemeral node (https://tailscale.com/s/ephemeral-nodes).
	Ephemeral bool

	// AuthKey, if non-empty, is the auth key to create the node
	// and will be preferred over the TS_AUTHKEY environment
	// variable. If the node is already created (from state
	// previously stored in Store), then this field is not
	// used.
	AuthKey string

	// ClientSecret, if non-empty, is the OAuth client secret
	// that will be used to generate authkeys via OAuth. It
	// will be preferred over the TS_CLIENT_SECRET environment
	// variable. If the node is already created (from state
	// previously stored in Store), then this field is not
	// used.
	ClientSecret string

	// ClientID, if non-empty, is the client ID used to generate
	// authkeys via workload identity federation. It will be
	// preferred over the TS_CLIENT_ID environment variable.
	// If the node is already created (from state previously
	// stored in Store), then this field is not used.
	ClientID string

	// IDToken, if non-empty, is the ID token from the identity
	// provider to exchange with the control server for workload
	// identity federation. It will be preferred over the
	// TS_ID_TOKEN environment variable. If the node is already
	// created (from state previously stored in Store), then this
	// field is not used.
	IDToken string

	// Audience, if non-empty, is the audience to use when requesting
	// an ID token from a well-known identity provider to exchange
	// with the control server for workload identity federation. It
	// will be preferred over the TS_AUDIENCE environment variable. If
	// the node is already created (from state previously stored in Store),
	// then this field is not used.
	Audience string

	// ControlURL optionally specifies the coordination server URL.
	// If empty, the Tailscale default is used.
	ControlURL string

	// RunWebClient, if true, runs a client for managing this node over
	// its Tailscale interface on port 5252.
	RunWebClient bool

	// Port is the UDP port to listen on for WireGuard and peer-to-peer
	// traffic. If zero, a port is automatically selected. Leave this
	// field at zero unless you know what you are doing.
	Port uint16

	// AdvertiseTags specifies tags that should be applied to this node, for
	// purposes of ACL enforcement. These can be referenced from the ACL policy
	// document. Note that advertising a tag on the client doesn't guarantee
	// that the control server will allow the node to adopt that tag.
	AdvertiseTags []string

	// Tun, if non-nil, specifies a custom tun.Device to use for packet I/O.
	//
	// This field must be set before calling Start.
	Tun tun.Device

	initOnce             sync.Once
	initErr              error
	lb                   *ipnlocal.LocalBackend
	sys                  *tsd.System
	netstack             *netstack.Impl
	netMon               *netmon.Monitor
	rootPath             string // the state directory
	hostname             string
	shutdownCtx          context.Context
	shutdownCancel       context.CancelFunc
	proxyCred            string        // SOCKS5 proxy auth for loopbackListener
	localAPICred         string        // basic auth password for loopbackListener
	loopbackListener     net.Listener  // optional loopback for localapi and proxies
	localAPIListener     net.Listener  // in-memory, used by localClient
	localClient          *local.Client // in-memory
	localAPIServer       *http.Server
	resetServeConfigOnce sync.Once
	logbuffer            *filch.Filch
	logtail              *logtail.Logger
	logid                logid.PublicID

	mu                  sync.Mutex
	listeners           map[listenKey]*listener
	fallbackTCPHandlers set.HandleSet[FallbackTCPHandler]
	dialer              *tsdial.Dialer
	closed              bool
}

// FallbackTCPHandler describes the callback which
// conditionally handles an incoming TCP flow for the
// provided (src/port, dst/port) 4-tuple. These are registered
// as handlers of last resort, and are called only if no
// listener could handle the incoming flow.
//
// If the callback returns intercept=false, the flow is rejected.
//
// When intercept=true, the behavior depends on whether the returned handler
// is non-nil: if nil, the connection is rejected. If non-nil, handler takes
// over the TCP conn.
type FallbackTCPHandler func(src, dst netip.AddrPort) (handler func(net.Conn), intercept bool)

// Dial connects to the address on the tailnet.
// It will start the server if it has not been started yet.
func (s *Server) Dial(ctx context.Context, network, address string) (net.Conn, error) {
	if err := s.Start(); err != nil {
		return nil, err
	}
	if err := s.awaitRunning(ctx); err != nil {
		return nil, err
	}
	return s.dialer.UserDial(ctx, network, address)
}

// awaitRunning waits until the backend is in state Running.
// If the backend is in state Starting, it blocks until it reaches
// a terminal state (such as Stopped, NeedsMachineAuth)
// or the context expires.
func (s *Server) awaitRunning(ctx context.Context) error {
	st := s.lb.State()
	for {
		if err := ctx.Err(); err != nil {
			return err
		}
		switch st {
		case ipn.Running:
			return nil
		case ipn.NeedsLogin, ipn.Starting:
			// Even after LocalBackend.Start, the state machine is still briefly
			// in the "NeedsLogin" state. So treat that as also "Starting" and
			// wait for us to get out of that state.
			s.lb.WatchNotifications(ctx, ipn.NotifyInitialState, nil, func(n *ipn.Notify) (keepGoing bool) {
				if n.State != nil {
					st = *n.State
				}
				return st == ipn.NeedsLogin || st == ipn.Starting
			})
		default:
			return fmt.Errorf("tsnet: backend in state %v", st)
		}
	}
}

// HTTPClient returns an HTTP client that is configured to connect over Tailscale.
//
// This is useful if you need to have your tsnet services connect to other devices on
// your tailnet.
func (s *Server) HTTPClient() *http.Client {
	return &http.Client{
		Transport: &http.Transport{
			DialContext: s.Dial,
		},
	}
}

// LocalClient returns a LocalClient that speaks to s.
//
// It will start the server if it has not been started yet. If the server's
// already been started successfully, it doesn't return an error.
func (s *Server) LocalClient() (*local.Client, error) {
	if err := s.Start(); err != nil {
		return nil, err
	}
	return s.localClient, nil
}

// Loopback starts a routing server on a loopback address.
//
// The server has multiple functions.
//
// It can be used as a SOCKS5 proxy onto the tailnet.
// Authentication is required with the username "tsnet" and
// the value of proxyCred used as the password.
//
// The HTTP server also serves out the "LocalAPI" on /localapi.
// As the LocalAPI is powerful, access to endpoints requires BOTH passing a
// "Sec-Tailscale: localapi" HTTP header and passing localAPICred as basic auth.
//
// If you only need to use the LocalAPI from Go, then prefer LocalClient
// as it does not require communication via TCP.
func (s *Server) Loopback() (addr string, proxyCred, localAPICred string, err error) {
	if err := s.Start(); err != nil {
		return "", "", "", err
	}

	if s.loopbackListener == nil {
		var proxyCred [16]byte
		if _, err := crand.Read(proxyCred[:]); err != nil {
			return "", "", "", err
		}
		s.proxyCred = hex.EncodeToString(proxyCred[:])

		var cred [16]byte
		if _, err := crand.Read(cred[:]); err != nil {
			return "", "", "", err
		}
		s.localAPICred = hex.EncodeToString(cred[:])

		ln, err := net.Listen("tcp", "127.0.0.1:0")
		if err != nil {
			return "", "", "", err
		}
		s.loopbackListener = ln

		socksLn, httpLn := proxymux.SplitSOCKSAndHTTP(ln)

		// TODO: add HTTP proxy support. Probably requires factoring
		// out the CONNECT code from tailscaled/proxy.go that uses
		// httputil.ReverseProxy and adding auth support.
		go func() {
			lah := localapi.NewHandler(localapi.HandlerConfig{
				Actor:    ipnauth.Self,
				Backend:  s.lb,
				Logf:     s.logf,
				LogID:    s.logid,
				EventBus: s.sys.Bus.Get(),
			})
			lah.PermitWrite = true
			lah.PermitRead = true
			lah.RequiredPassword = s.localAPICred
			h := &localSecHandler{h: lah, cred: s.localAPICred}

			if err := http.Serve(httpLn, h); err != nil {
				s.logf("localapi tcp serve error: %v", err)
			}
		}()
		s5l := logger.WithPrefix(s.logf, "socks5: ")
		s5s := &socks5.Server{
			Logf:     s5l,
			Dialer:   s.dialer.UserDial,
			Username: "tsnet",
			Password: s.proxyCred,
		}
		go func() {
			s5l("SOCKS5 server exited: %v", s5s.Serve(socksLn))
		}()
	}

	lbAddr := s.loopbackListener.Addr()
	if lbAddr == nil {
		// https://github.com/tailscale/tailscale/issues/7488
		panic("loopbackListener has no Addr")
	}
	return lbAddr.String(), s.proxyCred, s.localAPICred, nil
}

type localSecHandler struct {
	h    http.Handler
	cred string
}

func (h *localSecHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Header.Get("Sec-Tailscale") != "localapi" {
		w.WriteHeader(403)
		io.WriteString(w, "missing 'Sec-Tailscale: localapi' header")
		return
	}
	h.h.ServeHTTP(w, r)
}

// Start connects the server to the tailnet.
// Optional: any calls to Dial/Listen will also call Start.
func (s *Server) Start() error {
	hostinfo.SetPackage("tsnet")
	s.initOnce.Do(s.doInit)
	return s.initErr
}

// Up connects the server to the tailnet and waits until it is running.
// On success it returns the current status, including a Tailscale IP address.
func (s *Server) Up(ctx context.Context) (*ipnstate.Status, error) {
	lc, err := s.LocalClient() // calls Start
	if err != nil {
		return nil, fmt.Errorf("tsnet.Up: %w", err)
	}

	watcher, err := lc.WatchIPNBus(ctx, ipn.NotifyInitialState)
	if err != nil {
		return nil, fmt.Errorf("tsnet.Up: %w", err)
	}
	defer watcher.Close()

	for {
		n, err := watcher.Next()
		if err != nil {
			return nil, fmt.Errorf("tsnet.Up: %w", err)
		}
		if n.ErrMessage != nil {
			return nil, fmt.Errorf("tsnet.Up: backend: %s", *n.ErrMessage)
		}
		if st := n.State; st != nil {
			if *st == ipn.Running {
				status, err := lc.Status(ctx)
				if err != nil {
					return nil, fmt.Errorf("tsnet.Up: %w", err)
				}
				if len(status.TailscaleIPs) == 0 {
					return nil, errors.New("tsnet.Up: running, but no ip")
				}

				// The first time Up is run, clear the persisted serve config.
				// We do this to prevent messy interactions with stale config in
				// the face of code changes.
				var srvResetErr error
				s.resetServeConfigOnce.Do(func() {
					srvResetErr = lc.SetServeConfig(ctx, new(ipn.ServeConfig))
				})
				if srvResetErr != nil {
					return nil, fmt.Errorf("tsnet.Up: clearing serve config: %w", err)
				}

				return status, nil
			}
			// TODO: in the future, return an error on ipn.NeedsLogin
			// and ipn.NeedsMachineAuth to improve the UX of trying
			// out the tsnet package.
			//
			// Unfortunately today, even when using an AuthKey we
			// briefly see these states. It would be nice to fix.
		}
	}
}

// Close stops the server.
//
// It must not be called before or concurrently with Start.
func (s *Server) Close() error {
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.closed {
		return fmt.Errorf("tsnet: %w", net.ErrClosed)
	}
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		// Perform a best-effort final flush.
		if s.logtail != nil {
			s.logtail.Shutdown(ctx)
		}
		if s.logbuffer != nil {
			s.logbuffer.Close()
		}
	}()
	wg.Add(1)
	go func() {
		defer wg.Done()
		if s.localAPIServer != nil {
			s.localAPIServer.Shutdown(ctx)
		}
	}()

	if s.netstack != nil {
		s.netstack.Close()
		s.netstack = nil
	}
	if s.shutdownCancel != nil {
		s.shutdownCancel()
	}
	if s.lb != nil {
		s.lb.Shutdown()
	}
	if s.netMon != nil {
		s.netMon.Close()
	}
	if s.dialer != nil {
		s.dialer.Close()
	}
	if s.localAPIListener != nil {
		s.localAPIListener.Close()
	}
	if s.loopbackListener != nil {
		s.loopbackListener.Close()
	}

	for _, ln := range s.listeners {
		ln.closeLocked()
	}
	wg.Wait()
	s.sys.Bus.Get().Close()
	s.closed = true
	return nil
}

func (s *Server) doInit() {
	s.shutdownCtx, s.shutdownCancel = context.WithCancel(context.Background())
	if err := s.start(); err != nil {
		s.initErr = fmt.Errorf("tsnet: %w", err)
	}
}

// CertDomains returns the list of domains for which the server can
// provide TLS certificates. These are also the DNS names for the
// Server.
// If the server is not running, it returns nil.
func (s *Server) CertDomains() []string {
	nm := s.lb.NetMap()
	if nm == nil {
		return nil
	}
	return slices.Clone(nm.DNS.CertDomains)
}

// TailscaleIPs returns IPv4 and IPv6 addresses for this node. If the node
// has not yet joined a tailnet or is otherwise unaware of its own IP addresses,
// the returned ip4, ip6 will be !netip.IsValid().
func (s *Server) TailscaleIPs() (ip4, ip6 netip.Addr) {
	nm := s.lb.NetMap()
	if nm == nil {
		return
	}
	addrs := nm.GetAddresses()
	for _, addr := range addrs.All() {
		ip := addr.Addr()
		if ip.Is6() {
			ip6 = ip
		}
		if ip.Is4() {
			ip4 = ip
		}
	}

	return ip4, ip6
}

// LogtailWriter returns an [io.Writer] that writes to Tailscale's logging service and will be only visible to Tailscale's
// support team. Logs written there cannot be retrieved by the user. This method always returns a non-nil value.
func (s *Server) LogtailWriter() io.Writer {
	if s.logtail == nil {
		return io.Discard
	}

	return s.logtail
}

func (s *Server) getAuthKey() string {
	if v := s.AuthKey; v != "" {
		return v
	}
	if v := os.Getenv("TS_AUTHKEY"); v != "" {
		return v
	}
	return os.Getenv("TS_AUTH_KEY")
}

func (s *Server) getClientSecret() string {
	if v := s.ClientSecret; v != "" {
		return v
	}
	return os.Getenv("TS_CLIENT_SECRET")
}

func (s *Server) getClientID() string {
	if v := s.ClientID; v != "" {
		return v
	}
	return os.Getenv("TS_CLIENT_ID")
}

func (s *Server) getIDToken() string {
	if v := s.IDToken; v != "" {
		return v
	}
	return os.Getenv("TS_ID_TOKEN")
}

func (s *Server) getAudience() string {
	if v := s.Audience; v != "" {
		return v
	}
	return os.Getenv("TS_AUDIENCE")
}

func (s *Server) start() (reterr error) {
	var closePool closeOnErrorPool
	defer closePool.closeAllIfError(&reterr)

	exe, err := os.Executable()
	if err != nil {
		switch runtime.GOOS {
		case "js", "wasip1":
			// These platforms don't implement os.Executable (at least as of Go
			// 1.21), but we don't really care much: it's only used as a default
			// directory and hostname when they're not supplied. But we can fall
			// back to "tsnet" as well.
			exe = "tsnet"
		case "ios":
			// When compiled as a framework (via TailscaleKit in libtailscale),
			// os.Executable() returns an error, so fall back to "tsnet" there
			// too.
			exe = "tsnet"
		default:
			return err
		}
	}
	prog := strings.TrimSuffix(strings.ToLower(filepath.Base(exe)), ".exe")

	s.hostname = s.Hostname
	if s.hostname == "" {
		s.hostname = prog
	}

	s.rootPath = s.Dir
	if s.Store != nil {
		_, isMemStore := s.Store.(*mem.Store)
		if isMemStore && !s.Ephemeral {
			return fmt.Errorf("in-memory store is only supported for Ephemeral nodes")
		}
	}

	if s.rootPath == "" {
		confDir, err := os.UserConfigDir()
		if err != nil {
			return err
		}
		s.rootPath = filepath.Join(confDir, "tsnet-"+prog)
	}
	if err := os.MkdirAll(s.rootPath, 0700); err != nil {
		return err
	}
	if fi, err := os.Stat(s.rootPath); err != nil {
		return err
	} else if !fi.IsDir() {
		return fmt.Errorf("%v is not a directory", s.rootPath)
	}

	tsLogf := func(format string, a ...any) {
		if s.logtail != nil {
			s.logtail.Logf(format, a...)
		}
		if s.Logf == nil {
			return
		}
		s.Logf(format, a...)
	}

	sys := tsd.NewSystem()
	s.sys = sys
	if err := s.startLogger(&closePool, sys.HealthTracker.Get(), tsLogf); err != nil {
		return err
	}

	s.netMon, err = netmon.New(sys.Bus.Get(), tsLogf)
	if err != nil {
		return err
	}
	closePool.add(s.netMon)

	s.dialer = &tsdial.Dialer{Logf: tsLogf} // mutated below (before used)
	s.dialer.SetBus(sys.Bus.Get())
	eng, err := wgengine.NewUserspaceEngine(tsLogf, wgengine.Config{
		Tun:           s.Tun,
		EventBus:      sys.Bus.Get(),
		ListenPort:    s.Port,
		NetMon:        s.netMon,
		Dialer:        s.dialer,
		SetSubsystem:  sys.Set,
		ControlKnobs:  sys.ControlKnobs(),
		HealthTracker: sys.HealthTracker.Get(),
		Metrics:       sys.UserMetricsRegistry(),
	})
	if err != nil {
		return err
	}
	closePool.add(s.dialer)
	sys.Set(eng)
	sys.HealthTracker.Get().SetMetricsRegistry(sys.UserMetricsRegistry())

	// TODO(oxtoacart): do we need to support Taildrive on tsnet, and if so, how?
	ns, err := netstack.Create(tsLogf, sys.Tun.Get(), eng, sys.MagicSock.Get(), s.dialer, sys.DNSManager.Get(), sys.ProxyMapper())
	if err != nil {
		return fmt.Errorf("netstack.Create: %w", err)
	}
	sys.Tun.Get().Start()
	sys.Set(ns)
	if s.Tun == nil {
		// Only process packets in netstack when using the default fake TUN.
		// When a TUN is provided, let packets flow through it instead.
		ns.ProcessLocalIPs = true
		ns.ProcessSubnets = true
	} else {
		// When using a TUN, check gVisor for registered endpoints to handle
		// packets for tsnet listeners and outbound connection replies.
		ns.CheckLocalTransportEndpoints = true
	}
	ns.GetTCPHandlerForFlow = s.getTCPHandlerForFlow
	ns.GetUDPHandlerForFlow = s.getUDPHandlerForFlow
	s.netstack = ns
	s.dialer.UseNetstackForIP = func(ip netip.Addr) bool {
		_, ok := eng.PeerForIP(ip)
		return ok
	}
	s.dialer.NetstackDialTCP = func(ctx context.Context, dst netip.AddrPort) (net.Conn, error) {
		// Note: don't just return ns.DialContextTCP or we'll return
		// *gonet.TCPConn(nil) instead of a nil interface which trips up
		// callers.
		v4, v6 := s.TailscaleIPs()
		src := bools.IfElse(dst.Addr().Is6(), v6, v4)
		tcpConn, err := ns.DialContextTCPWithBind(ctx, src, dst)
		if err != nil {
			return nil, err
		}
		return tcpConn, nil
	}
	s.dialer.NetstackDialUDP = func(ctx context.Context, dst netip.AddrPort) (net.Conn, error) {
		// Note: don't just return ns.DialContextUDP or we'll return
		// *gonet.UDPConn(nil) instead of a nil interface which trips up
		// callers.
		v4, v6 := s.TailscaleIPs()
		src := bools.IfElse(dst.Addr().Is6(), v6, v4)
		udpConn, err := ns.DialContextUDPWithBind(ctx, src, dst)
		if err != nil {
			return nil, err
		}
		return udpConn, nil
	}

	if s.Store == nil {
		stateFile := filepath.Join(s.rootPath, "tailscaled.state")
		s.logf("tsnet running state path %s", stateFile)
		s.Store, err = store.New(tsLogf, stateFile)
		if err != nil {
			return err
		}
	}
	sys.Set(s.Store)

	loginFlags := controlclient.LoginDefault
	if s.Ephemeral {
		loginFlags = controlclient.LoginEphemeral
	}
	lb, err := ipnlocal.NewLocalBackend(tsLogf, s.logid, sys, loginFlags|controlclient.LocalBackendStartKeyOSNeutral)
	if err != nil {
		return fmt.Errorf("NewLocalBackend: %v", err)
	}
	lb.SetTCPHandlerForFunnelFlow(s.getTCPHandlerForFunnelFlow)
	lb.SetVarRoot(s.rootPath)
	s.logf("tsnet starting with hostname %q, varRoot %q", s.hostname, s.rootPath)
	s.lb = lb
	if err := ns.Start(lb); err != nil {
		return fmt.Errorf("failed to start netstack: %w", err)
	}
	closePool.addFunc(func() { s.lb.Shutdown() })
	prefs := ipn.NewPrefs()
	prefs.Hostname = s.hostname
	prefs.WantRunning = true
	prefs.ControlURL = s.ControlURL
	prefs.RunWebClient = s.RunWebClient
	prefs.AdvertiseTags = s.AdvertiseTags
	authKey, err := s.resolveAuthKey()
	if err != nil {
		return fmt.Errorf("error resolving auth key: %w", err)
	}
	err = lb.Start(ipn.Options{
		UpdatePrefs: prefs,
		AuthKey:     authKey,
	})
	if err != nil {
		return fmt.Errorf("starting backend: %w", err)
	}
	st := lb.State()
	if st == ipn.NeedsLogin || envknob.Bool("TSNET_FORCE_LOGIN") {
		s.logf("LocalBackend state is %v; running StartLoginInteractive...", st)
		if err := s.lb.StartLoginInteractive(s.shutdownCtx); err != nil {
			return fmt.Errorf("StartLoginInteractive: %w", err)
		}
	} else if authKey != "" {
		s.logf("Authkey is set; but state is %v. Ignoring authkey. Re-run with TSNET_FORCE_LOGIN=1 to force use of authkey.", st)
	}
	go s.printAuthURLLoop()

	// Run the localapi handler, to allow fetching LetsEncrypt certs.
	lah := localapi.NewHandler(localapi.HandlerConfig{
		Actor:    ipnauth.Self,
		Backend:  lb,
		Logf:     tsLogf,
		LogID:    s.logid,
		EventBus: sys.Bus.Get(),
	})
	lah.PermitWrite = true
	lah.PermitRead = true

	// Create an in-process listener.
	// nettest.Listen provides a in-memory pipe based implementation for net.Conn.
	lal := memnet.Listen("local-tailscaled.sock:80")
	s.localAPIListener = lal
	s.localClient = &local.Client{Dial: lal.Dial}
	s.localAPIServer = &http.Server{Handler: lah}
	s.lb.ConfigureWebClient(s.localClient)
	go func() {
		if err := s.localAPIServer.Serve(lal); err != nil && err != http.ErrServerClosed {
			s.logf("localapi serve error: %v", err)
		}
	}()
	closePool.add(s.localAPIListener)
	return nil
}

func (s *Server) resolveAuthKey() (string, error) {
	authKey := s.getAuthKey()
	var err error
	// Try to use an OAuth secret to generate an auth key if that functionality
	// is available.
	resolveViaOAuth, oauthOk := tailscale.HookResolveAuthKey.GetOk()
	if oauthOk {
		clientSecret := authKey
		if authKey == "" {
			clientSecret = s.getClientSecret()
		}
		authKey, err = resolveViaOAuth(s.shutdownCtx, clientSecret, s.AdvertiseTags)
		if err != nil {
			return "", err
		}
	}
	// Try to resolve the auth key via workload identity federation if that functionality
	// is available and no auth key is yet determined.
	resolveViaWIF, wifOk := tailscale.HookResolveAuthKeyViaWIF.GetOk()
	if wifOk && authKey == "" {
		clientID := s.getClientID()
		idToken := s.getIDToken()
		audience := s.getAudience()
		if clientID != "" && idToken == "" && audience == "" {
			return "", fmt.Errorf("client ID for workload identity federation found, but ID token and audience are empty")
		}
		if idToken != "" && audience != "" {
			return "", fmt.Errorf("only one of ID token and audience should be for workload identity federation")
		}
		if clientID == "" {
			if idToken != "" {
				return "", fmt.Errorf("ID token for workload identity federation found, but client ID is empty")
			}
			if audience != "" {
				return "", fmt.Errorf("audience for workload identity federation found, but client ID is empty")
			}
		}
		authKey, err = resolveViaWIF(s.shutdownCtx, s.ControlURL, clientID, idToken, audience, s.AdvertiseTags)
		if err != nil {
			return "", err
		}
	}
	return authKey, nil
}

func (s *Server) startLogger(closePool *closeOnErrorPool, health *health.Tracker, tsLogf logger.Logf) error {
	if testenv.InTest() {
		return nil
	}
	cfgPath := filepath.Join(s.rootPath, "tailscaled.log.conf")
	lpc, err := logpolicy.ConfigFromFile(cfgPath)
	switch {
	case os.IsNotExist(err):
		lpc = logpolicy.NewConfig(logtail.CollectionNode)
		if err := lpc.Save(cfgPath); err != nil {
			return fmt.Errorf("logpolicy.Config.Save for %v: %w", cfgPath, err)
		}
	case err != nil:
		return fmt.Errorf("logpolicy.LoadConfig for %v: %w", cfgPath, err)
	}
	if err := lpc.Validate(logtail.CollectionNode); err != nil {
		return fmt.Errorf("logpolicy.Config.Validate for %v: %w", cfgPath, err)
	}
	s.logid = lpc.PublicID

	s.logbuffer, err = filch.New(filepath.Join(s.rootPath, "tailscaled"), filch.Options{ReplaceStderr: false})
	if err != nil {
		return fmt.Errorf("error creating filch: %w", err)
	}
	closePool.add(s.logbuffer)
	c := logtail.Config{
		Collection:   lpc.Collection,
		PrivateID:    lpc.PrivateID,
		Stderr:       io.Discard, // log everything to Buffer
		Buffer:       s.logbuffer,
		CompressLogs: true,
		Bus:          s.sys.Bus.Get(),
		HTTPC:        &http.Client{Transport: logpolicy.NewLogtailTransport(logtail.DefaultHost, s.netMon, health, tsLogf)},
		MetricsDelta: clientmetric.EncodeLogTailMetricsDelta,
	}
	s.logtail = logtail.NewLogger(c, tsLogf)
	closePool.addFunc(func() { s.logtail.Shutdown(context.Background()) })
	return nil
}

type closeOnErrorPool []func()

func (p *closeOnErrorPool) add(c io.Closer)   { *p = append(*p, func() { c.Close() }) }
func (p *closeOnErrorPool) addFunc(fn func()) { *p = append(*p, fn) }
func (p closeOnErrorPool) closeAllIfError(errp *error) {
	if *errp != nil {
		for _, closeFn := range p {
			closeFn()
		}
	}
}

func (s *Server) logf(format string, a ...any) {
	if s.logtail != nil {
		s.logtail.Logf(format, a...)
	}
	if s.UserLogf != nil {
		s.UserLogf(format, a...)
		return
	}
	log.Printf(format, a...)
}

// printAuthURLLoop loops once every few seconds while the server is still running and
// is in NeedsLogin state, printing out the auth URL.
func (s *Server) printAuthURLLoop() {
	for {
		if s.shutdownCtx.Err() != nil {
			return
		}
		if st := s.lb.State(); st != ipn.NeedsLogin && st != ipn.NoState {
			s.logf("AuthLoop: state is %v; done", st)
			return
		}
		st := s.lb.StatusWithoutPeers()
		if st.AuthURL != "" {
			s.logf("To start this tsnet server, restart with TS_AUTHKEY set, or go to: %s", st.AuthURL)
		}
		select {
		case <-time.After(5 * time.Second):
		case <-s.shutdownCtx.Done():
			return
		}
	}
}

// networkForFamily returns one of "tcp4", "tcp6", "udp4", or "udp6".
//
// netBase is "tcp" or "udp" (without any '4' or '6' suffix).
func networkForFamily(netBase string, is6 bool) string {
	switch netBase {
	case "tcp":
		if is6 {
			return "tcp6"
		}
		return "tcp4"
	case "udp":
		if is6 {
			return "udp6"
		}
		return "udp4"
	}
	panic("unexpected")
}

// listenerForDstAddr returns a listener for the provided network and
// destination IP/port. It matches from most specific to least specific.
// For example:
//
//   - ("tcp4", IP, port)
//   - ("tcp", IP, port)
//   - ("tcp4", "", port)
//   - ("tcp", "", port)
//
// The netBase is "tcp" or "udp" (without any '4' or '6' suffix).
//
// Listeners which do not specify an IP address will match for traffic
// for the local node (that is, a destination address of the IPv4 or
// IPv6 address of this node) only. To listen for traffic on other addresses
// such as those routed inbound via subnet routes, explicitly specify
// the listening address or use RegisterFallbackTCPHandler.
func (s *Server) listenerForDstAddr(netBase string, dst netip.AddrPort, funnel bool) (_ *listener, ok bool) {
	s.mu.Lock()
	defer s.mu.Unlock()

	// Search for a listener with the specified IP
	for _, net := range [2]string{
		networkForFamily(netBase, dst.Addr().Is6()),
		netBase,
	} {
		if ln, ok := s.listeners[listenKey{net, dst.Addr(), dst.Port(), funnel}]; ok {
			return ln, true
		}
	}

	// Search for a listener without an IP if the destination was
	// one of the native IPs of the node.
	if ip4, ip6 := s.TailscaleIPs(); dst.Addr() == ip4 || dst.Addr() == ip6 {
		for _, net := range [2]string{
			networkForFamily(netBase, dst.Addr().Is6()),
			netBase,
		} {
			if ln, ok := s.listeners[listenKey{net, netip.Addr{}, dst.Port(), funnel}]; ok {
				return ln, true
			}
		}
	}

	return nil, false
}

func (s *Server) getTCPHandlerForFunnelFlow(src netip.AddrPort, dstPort uint16) (handler func(net.Conn)) {
	ipv4, ipv6 := s.TailscaleIPs()
	var dst netip.AddrPort
	if src.Addr().Is4() {
		if !ipv4.IsValid() {
			return nil
		}
		dst = netip.AddrPortFrom(ipv4, dstPort)
	} else {
		if !ipv6.IsValid() {
			return nil
		}
		dst = netip.AddrPortFrom(ipv6, dstPort)
	}
	ln, ok := s.listenerForDstAddr("tcp", dst, true)
	if !ok {
		return nil
	}
	return ln.handle
}

func (s *Server) getTCPHandlerForFlow(src, dst netip.AddrPort) (handler func(net.Conn), intercept bool) {
	ln, ok := s.listenerForDstAddr("tcp", dst, false)
	if !ok {
		s.mu.Lock()
		defer s.mu.Unlock()
		for _, handler := range s.fallbackTCPHandlers {
			connHandler, intercept := handler(src, dst)
			if intercept {
				return connHandler, intercept
			}
		}
		return nil, true // don't handle, don't forward to localhost
	}
	return ln.handle, true
}

func (s *Server) getUDPHandlerForFlow(src, dst netip.AddrPort) (handler func(nettype.ConnPacketConn), intercept bool) {
	ln, ok := s.listenerForDstAddr("udp", dst, false)
	if !ok {
		return nil, true // don't handle, don't forward to localhost
	}
	return func(c nettype.ConnPacketConn) { ln.handle(c) }, true
}

// Listen announces only on the Tailscale network.
// It will start the server if it has not been started yet.
//
// Listeners which do not specify an IP address will match for traffic
// for the local node (that is, a destination address of the IPv4 or
// IPv6 address of this node) only. To listen for traffic on other addresses
// such as those routed inbound via subnet routes, explicitly specify
// the listening address or use RegisterFallbackTCPHandler.
func (s *Server) Listen(network, addr string) (net.Listener, error) {
	return s.listen(network, addr, listenOnTailnet)
}

// ListenPacket announces on the Tailscale network.
//
// The network must be "udp", "udp4" or "udp6". The addr must be of the form
// "ip:port" (or "[ip]:port") where ip is a valid IPv4 or IPv6 address
// corresponding to "udp4" or "udp6" respectively. IP must be specified.
//
// If s has not been started yet, it will be started.
func (s *Server) ListenPacket(network, addr string) (net.PacketConn, error) {
	ap, err := resolveListenAddr(network, addr)
	if err != nil {
		return nil, err
	}
	if !ap.Addr().IsValid() {
		return nil, fmt.Errorf("tsnet.ListenPacket(%q, %q): address must be a valid IP", network, addr)
	}
	if network == "udp" {
		if ap.Addr().Is4() {
			network = "udp4"
		} else {
			network = "udp6"
		}
	}

	netLn, err := s.listen(network, addr, listenOnTailnet)
	if err != nil {
		return nil, err
	}
	ln := netLn.(*listener)

	pc, err := s.netstack.ListenPacket(network, ap.String())
	if err != nil {
		ln.Close()
		return nil, err
	}

	return &udpPacketConn{
		PacketConn: pc,
		ln:         ln,
	}, nil
}

// udpPacketConn wraps a net.PacketConn to unregister from s.listeners on Close.
type udpPacketConn struct {
	net.PacketConn
	ln *listener
}

func (c *udpPacketConn) Close() error {
	c.ln.Close()
	return c.PacketConn.Close()
}

// ListenTLS announces only on the Tailscale network.
// It returns a TLS listener wrapping the tsnet listener.
// It will start the server if it has not been started yet.
func (s *Server) ListenTLS(network, addr string) (net.Listener, error) {
	if network != "tcp" {
		return nil, fmt.Errorf("ListenTLS(%q, %q): only tcp is supported", network, addr)
	}
	ctx := context.Background()
	st, err := s.Up(ctx)
	if err != nil {
		return nil, err
	}
	if !st.CurrentTailnet.MagicDNSEnabled {
		return nil, errors.New("tsnet: you must enable MagicDNS in the DNS page of the admin panel to proceed. See https://tailscale.com/s/https")
	}
	if len(st.CertDomains) == 0 {
		return nil, errors.New("tsnet: you must enable HTTPS in the admin panel to proceed. See https://tailscale.com/s/https")
	}

	ln, err := s.listen(network, addr, listenOnTailnet)
	if err != nil {
		return nil, err
	}
	return tls.NewListener(ln, &tls.Config{
		GetCertificate: s.getCert,
	}), nil
}

// RegisterFallbackTCPHandler registers a callback which will be called
// to handle a TCP flow to this tsnet node, for which no listeners will handle.
//
// If multiple fallback handlers are registered, they will be called in an
// undefined order. See FallbackTCPHandler for details on handling a flow.
//
// The returned function can be used to deregister this callback.
func (s *Server) RegisterFallbackTCPHandler(cb FallbackTCPHandler) func() {
	s.mu.Lock()
	defer s.mu.Unlock()
	hnd := s.fallbackTCPHandlers.Add(cb)
	return func() {
		s.mu.Lock()
		defer s.mu.Unlock()
		delete(s.fallbackTCPHandlers, hnd)
	}
}

// getCert is the GetCertificate function used by ListenTLS.
//
// It calls GetCertificate on the localClient, passing in the ClientHelloInfo.
// For testing, if s.getCertForTesting is set, it will call that instead.
func (s *Server) getCert(hi *tls.ClientHelloInfo) (*tls.Certificate, error) {
	lc, err := s.LocalClient()
	if err != nil {
		return nil, err
	}
	return lc.GetCertificate(hi)
}

// FunnelOption is an option passed to ListenFunnel to configure the listener.
type FunnelOption interface {
	funnelOption()
}

type funnelOnly struct{}

func (funnelOnly) funnelOption() {}

// FunnelOnly configures the listener to only respond to connections from Tailscale Funnel.
// The local tailnet will not be able to connect to the listener.
func FunnelOnly() FunnelOption { return funnelOnly{} }

type funnelTLSConfig struct{ conf *tls.Config }

func (f funnelTLSConfig) funnelOption() {}

// FunnelTLSConfig configures the TLS configuration for [Server.ListenFunnel]
//
// This is rarely needed but can permit requiring client certificates, specific
// ciphers suites, etc.
//
// The provided conf should at least be able to get a certificate, setting
// GetCertificate, Certificates or GetConfigForClient appropriately.
// The most common configuration is to set GetCertificate to
// Server.LocalClient's GetCertificate method.
//
// Unless [FunnelOnly] is also used, the configuration is also used for
// in-tailnet connections that don't arrive over Funnel.
func FunnelTLSConfig(conf *tls.Config) FunnelOption {
	return funnelTLSConfig{conf: conf}
}

// ListenFunnel announces on the public internet using Tailscale Funnel.
//
// It also by default listens on your local tailnet, so connections can
// come from either inside or outside your network. To restrict connections
// to be just from the internet, use the FunnelOnly option.
//
// Currently (2023-03-10), Funnel only supports TCP on ports 443, 8443, and 10000.
// The supported host name is limited to that configured for the tsnet.Server.
// As such, the standard way to create funnel is:
//
//	s.ListenFunnel("tcp", ":443")
//
// and the only other supported addrs currently are ":8443" and ":10000".
//
// It will start the server if it has not been started yet.
func (s *Server) ListenFunnel(network, addr string, opts ...FunnelOption) (net.Listener, error) {
	if network != "tcp" {
		return nil, fmt.Errorf("ListenFunnel(%q, %q): only tcp is supported", network, addr)
	}
	host, portStr, err := net.SplitHostPort(addr)
	if err != nil {
		return nil, err
	}
	if host != "" {
		return nil, fmt.Errorf("ListenFunnel(%q, %q): host must be empty", network, addr)
	}
	port, err := strconv.ParseUint(portStr, 10, 16)
	if err != nil {
		return nil, err
	}

	// Process, validate opts.
	lnOn := listenOnBoth
	var tlsConfig *tls.Config
	for _, opt := range opts {
		switch v := opt.(type) {
		case funnelTLSConfig:
			if v.conf == nil {
				return nil, errors.New("invalid nil FunnelTLSConfig")
			}
			tlsConfig = v.conf
		case funnelOnly:
			lnOn = listenOnFunnel
		default:
			return nil, fmt.Errorf("unknown opts FunnelOption type %T", v)
		}
	}
	if tlsConfig == nil {
		tlsConfig = &tls.Config{GetCertificate: s.getCert}
	}

	ctx := context.Background()
	st, err := s.Up(ctx)
	if err != nil {
		return nil, err
	}
	// TODO(sonia,tailscale/corp#10577): We may want to use the interactive enable
	// flow here instead of CheckFunnelAccess to allow the user to turn on Funnel
	// if not already on. Specifically when running from a terminal.
	// See cli.serveEnv.verifyFunnelEnabled.
	if err := ipn.CheckFunnelAccess(uint16(port), st.Self); err != nil {
		return nil, err
	}

	lc := s.localClient

	// May not have funnel enabled. Enable it.
	srvConfig, err := lc.GetServeConfig(ctx)
	if err != nil {
		return nil, err
	}
	if srvConfig == nil {
		srvConfig = &ipn.ServeConfig{}
	}
	if len(st.CertDomains) == 0 {
		return nil, errors.New("Funnel not available; HTTPS must be enabled. See https://tailscale.com/s/https")
	}
	domain := st.CertDomains[0]
	hp := ipn.HostPort(domain + ":" + portStr)
	var cleanupOnClose func() error
	if !srvConfig.AllowFunnel[hp] {
		mak.Set(&srvConfig.AllowFunnel, hp, true)
		srvConfig.AllowFunnel[hp] = true
		if err := lc.SetServeConfig(ctx, srvConfig); err != nil {
			return nil, err
		}
		cleanupOnClose = func() error {
			sc, err := lc.GetServeConfig(ctx)
			if err != nil {
				return fmt.Errorf("cleaning config changes: %w", err)
			}
			if sc.AllowFunnel != nil {
				delete(sc.AllowFunnel, hp)
			}
			if err := lc.SetServeConfig(ctx, sc); err != nil {
				return fmt.Errorf("cleaning config changes: %w", err)
			}
			return nil
		}
	}

	// Start a funnel listener.
	ln, err := s.listen(network, addr, lnOn)
	if err != nil {
		return nil, err
	}
	ln = &cleanupListener{Listener: ln, cleanup: cleanupOnClose}
	return tls.NewListener(ln, tlsConfig), nil
}

// ServiceMode defines how a Service is run. Currently supported modes are:
//   - [ServiceModeTCP]
//   - [ServiceModeHTTP]
//
// For more information, see [Server.ListenService].
type ServiceMode interface {
	// network is the network this Service will advertise on. Per Go convention,
	// this should be lowercase, e.g. 'tcp'.
	network() string
}

// serviceModeWithPort is a convenience type to extract the port from
// ServiceMode types which have one.
type serviceModeWithPort interface {
	ServiceMode
	port() uint16
}

// ServiceModeTCP is used to configure a TCP Service via [Server.ListenService].
type ServiceModeTCP struct {
	// Port is the TCP port to advertise. If this Service needs to advertise
	// multiple ports, call ListenService multiple times.
	Port uint16

	// TerminateTLS means that TLS connections will be terminated before being
	// forwarded to the listener. In this case, the only server name indicator
	// (SNI) permitted is the Service's fully-qualified domain name.
	TerminateTLS bool

	// PROXYProtocolVersion indicates whether to send a PROXY protocol header
	// before forwarding the connection to the listener and which version of the
	// protocol to use.
	//
	// For more information, see
	// https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
	PROXYProtocolVersion int
}

func (ServiceModeTCP) network() string { return "tcp" }

func (m ServiceModeTCP) port() uint16 { return m.Port }

// ServiceModeHTTP is used to configure an HTTP Service via
// [Server.ListenService].
type ServiceModeHTTP struct {
	// Port is the TCP port to advertise. If this Service needs to advertise
	// multiple ports, call ListenService multiple times.
	Port uint16

	// HTTPS, if true, means that the listener should handle connections as
	// HTTPS connections. In this case, the only server name indicator (SNI)
	// permitted is the Service's fully-qualified domain name.
	HTTPS bool

	// AcceptAppCaps defines the app capabilities to forward to the server. The
	// keys in this map are the mount points for each set of capabilities.
	//
	// By example,
	//
	//  AcceptAppCaps: map[string][]string{
	// 	  "/":    {"example.com/cap/all-paths"},
	// 	  "/foo": {"example.com/cap/all-paths", "example.com/cap/foo"},
	//  }
	//
	// would forward example.com/cap/all-paths to all paths on the server and
	// example.com/cap/foo only to paths beginning with /foo.
	//
	// For more information on app capabilities, see
	// https://tailscale.com/kb/1537/grants-app-capabilities
	AcceptAppCaps map[string][]string

	// PROXYProtocolVersion indicates whether to send a PROXY protocol header
	// before forwarding the connection to the listener and which version of the
	// protocol to use.
	//
	// For more information, see
	// https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
	PROXYProtocol int
}

func (ServiceModeHTTP) network() string { return "tcp" }

func (m ServiceModeHTTP) port() uint16 { return m.Port }

func (m ServiceModeHTTP) capsMap() map[string][]tailcfg.PeerCapability {
	capsMap := map[string][]tailcfg.PeerCapability{}
	for path, capNames := range m.AcceptAppCaps {
		caps := make([]tailcfg.PeerCapability, 0, len(capNames))
		for _, c := range capNames {
			caps = append(caps, tailcfg.PeerCapability(c))
		}
		capsMap[path] = caps
	}
	return capsMap
}

// A ServiceListener is a network listener for a Tailscale Service. For more
// information about Services, see
// https://tailscale.com/kb/1552/tailscale-services
type ServiceListener struct {
	net.Listener
	addr addr

	// FQDN is the fully-qualifed domain name of this Service.
	FQDN string
}

// Addr returns the listener's network address. This will be the Service's
// fully-qualified domain name (FQDN) and the port.
//
// A hostname is not truly a network address, but Services listen on multiple
// addresses (the IPv4 and IPv6 virtual IPs).
func (sl ServiceListener) Addr() net.Addr {
	return sl.addr
}

// ErrUntaggedServiceHost is returned by ListenService when run on a node
// without any ACL tags. A node must use a tag-based identity to act as a
// Service host. For more information, see:
// https://tailscale.com/kb/1552/tailscale-services#prerequisites
var ErrUntaggedServiceHost = errors.New("service hosts must be tagged nodes")

// ListenService creates a network listener for a Tailscale Service. This will
// advertise this node as hosting the Service. Note that:
//   - Approval must still be granted by an admin or by ACL auto-approval rules.
//   - Service hosts must be tagged nodes.
//   - A valid Service host must advertise all ports defined for the Service.
//
// To advertise a Service with multiple ports, run ListenService multiple times.
// For more information about Services, see
// https://tailscale.com/kb/1552/tailscale-services
//
// This function will start the server if it is not already started.
func (s *Server) ListenService(name string, mode ServiceMode) (*ServiceListener, error) {
	if err := tailcfg.ServiceName(name).Validate(); err != nil {
		return nil, err
	}
	if mode == nil {
		return nil, errors.New("mode may not be nil")
	}
	svcName := name

	// TODO(hwh33,tailscale/corp#35859): support TUN mode

	ctx := context.Background()
	_, err := s.Up(ctx)
	if err != nil {
		return nil, err
	}

	st := s.lb.StatusWithoutPeers()
	if st.Self.Tags == nil || st.Self.Tags.Len() == 0 {
		return nil, ErrUntaggedServiceHost
	}

	advertisedServices := s.lb.Prefs().AdvertiseServices().AsSlice()
	if !slices.Contains(advertisedServices, svcName) {
		// TODO(hwh33,tailscale/corp#35860): clean these prefs up when (a) we
		// exit early due to error or (b) when the returned listener is closed.
		_, err = s.lb.EditPrefs(&ipn.MaskedPrefs{
			AdvertiseServicesSet: true,
			Prefs: ipn.Prefs{
				AdvertiseServices: append(advertisedServices, svcName),
			},
		})
		if err != nil {
			return nil, fmt.Errorf("updating advertised Services: %w", err)
		}
	}

	srvConfig := new(ipn.ServeConfig)
	sc, srvConfigETag, err := s.lb.ServeConfigETag()
	if err != nil {
		return nil, fmt.Errorf("fetching current serve config: %w", err)
	}
	if sc.Valid() {
		srvConfig = sc.AsStruct()
	}

	fqdn := tailcfg.ServiceName(svcName).WithoutPrefix() + "." + st.CurrentTailnet.MagicDNSSuffix

	// svcAddr is used to implement Addr() on the returned listener.
	svcAddr := addr{
		network: mode.network(),
		// A hostname is not a network address, but Services listen on
		// multiple addresses (the IPv4 and IPv6 virtual IPs), and there's
		// no clear winner here between the two. Therefore prefer the FQDN.
		//
		// In the case of TCP or HTTP Services, the port will be added below.
		addr: fqdn,
	}
	if m, ok := mode.(serviceModeWithPort); ok {
		if m.port() == 0 {
			return nil, errors.New("must specify a port to advertise")
		}
		svcAddr.addr += ":" + strconv.Itoa(int(m.port()))
	}

	// Start listening on a local TCP socket.
	ln, err := net.Listen("tcp", "localhost:0")
	if err != nil {
		return nil, fmt.Errorf("starting local listener: %w", err)
	}

	switch m := mode.(type) {
	case ServiceModeTCP:
		// Forward all connections from service-hostname:port to our socket.
		srvConfig.SetTCPForwardingForService(
			m.Port, ln.Addr().String(), m.TerminateTLS,
			tailcfg.ServiceName(svcName), m.PROXYProtocolVersion, st.CurrentTailnet.MagicDNSSuffix)
	case ServiceModeHTTP:
		// For HTTP Services, proxy all connections to our socket.
		mds := st.CurrentTailnet.MagicDNSSuffix
		haveRootHandler := false
		// We need to add a separate proxy for each mount point in the caps map.
		for path, caps := range m.capsMap() {
			if !strings.HasPrefix(path, "/") {
				path = "/" + path
			}
			h := ipn.HTTPHandler{
				AcceptAppCaps: caps,
				Proxy:         ln.Addr().String(),
			}
			if path == "/" {
				haveRootHandler = true
			} else {
				h.Proxy += path
			}
			srvConfig.SetWebHandler(&h, svcName, m.Port, path, m.HTTPS, mds)
		}
		// We always need a root handler.
		if !haveRootHandler {
			h := ipn.HTTPHandler{Proxy: ln.Addr().String()}
			srvConfig.SetWebHandler(&h, svcName, m.Port, "/", m.HTTPS, mds)
		}
	default:
		ln.Close()
		return nil, fmt.Errorf("unknown ServiceMode type %T", m)
	}

	if err := s.lb.SetServeConfig(srvConfig, srvConfigETag); err != nil {
		ln.Close()
		return nil, err
	}

	// TODO(hwh33,tailscale/corp#35860): clean up state (advertising prefs,
	// serve config changes) when the returned listener is closed.

	return &ServiceListener{
		Listener: ln,
		FQDN:     fqdn,
		addr:     svcAddr,
	}, nil
}

type listenOn string

const (
	listenOnTailnet = listenOn("listen-on-tailnet")
	listenOnFunnel  = listenOn("listen-on-funnel")
	listenOnBoth    = listenOn("listen-on-both")
)

// resolveListenAddr resolves a network and address into a netip.AddrPort. The
// returned netip.AddrPort.Addr will be the zero value if the address is empty.
// The port must be a valid port number. The caller is responsible for checking
// the network and address are valid.
//
// It resolves well-known port names and validates the address is a valid IP
// literal for the network.
func resolveListenAddr(network, addr string) (netip.AddrPort, error) {
	var zero netip.AddrPort
	host, portStr, err := net.SplitHostPort(addr)
	if err != nil {
		return zero, fmt.Errorf("tsnet: %w", err)
	}
	port, err := net.LookupPort(network, portStr)
	if err != nil || port < 0 || port > math.MaxUint16 {
		// LookupPort returns an error on out of range values so the bounds
		// checks on port should be unnecessary, but harmless. If they do
		// match, worst case this error message says "invalid port: <nil>".
		return zero, fmt.Errorf("invalid port: %w", err)
	}
	if host == "" {
		return netip.AddrPortFrom(netip.Addr{}, uint16(port)), nil
	}

	bindHostOrZero, err := netip.ParseAddr(host)
	if err != nil {
		return zero, fmt.Errorf("invalid Listen addr %q; host part must be empty or IP literal", host)
	}
	if strings.HasSuffix(network, "4") && !bindHostOrZero.Is4() {
		return zero, fmt.Errorf("invalid non-IPv4 addr %v for network %q", host, network)
	}
	if strings.HasSuffix(network, "6") && !bindHostOrZero.Is6() {
		return zero, fmt.Errorf("invalid non-IPv6 addr %v for network %q", host, network)
	}
	return netip.AddrPortFrom(bindHostOrZero, uint16(port)), nil
}

func (s *Server) listen(network, addr string, lnOn listenOn) (net.Listener, error) {
	switch network {
	case "", "tcp", "tcp4", "tcp6", "udp", "udp4", "udp6":
	default:
		return nil, errors.New("unsupported network type")
	}
	host, err := resolveListenAddr(network, addr)
	if err != nil {
		return nil, err
	}
	if err := s.Start(); err != nil {
		return nil, err
	}
	var keys []listenKey
	switch lnOn {
	case listenOnTailnet:
		keys = append(keys, listenKey{network, host.Addr(), host.Port(), false})
	case listenOnFunnel:
		keys = append(keys, listenKey{network, host.Addr(), host.Port(), true})
	case listenOnBoth:
		keys = append(keys, listenKey{network, host.Addr(), host.Port(), false})
		keys = append(keys, listenKey{network, host.Addr(), host.Port(), true})
	}

	ln := &listener{
		s:    s,
		keys: keys,
		addr: addr,

		closedc: make(chan struct{}),
		conn:    make(chan net.Conn),
	}

	// When using a TUN with TCP, create a gVisor TCP listener.
	if s.Tun != nil && (network == "" || network == "tcp" || network == "tcp4" || network == "tcp6") {
		var nsNetwork string
		nsAddr := host
		switch {
		case network == "tcp4" || network == "tcp6":
			nsNetwork = network
		case host.Addr().Is4():
			nsNetwork = "tcp4"
		case host.Addr().Is6():
			nsNetwork = "tcp6"
		default:
			// Wildcard address: use tcp6 for dual-stack (accepts both v4 and v6).
			nsNetwork = "tcp6"
			nsAddr = netip.AddrPortFrom(netip.IPv6Unspecified(), host.Port())
		}
		gonetLn, err := s.netstack.ListenTCP(nsNetwork, nsAddr.String())
		if err != nil {
			return nil, fmt.Errorf("tsnet: %w", err)
		}
		ln.gonetLn = gonetLn
	}

	s.mu.Lock()
	for _, key := range keys {
		if _, ok := s.listeners[key]; ok {
			s.mu.Unlock()
			if ln.gonetLn != nil {
				ln.gonetLn.Close()
			}
			return nil, fmt.Errorf("tsnet: listener already open for %s, %s", network, addr)
		}
	}
	if s.listeners == nil {
		s.listeners = make(map[listenKey]*listener)
	}
	for _, key := range keys {
		s.listeners[key] = ln
	}
	s.mu.Unlock()
	return ln, nil
}

// GetRootPath returns the root path of the tsnet server.
// This is where the state file and other data is stored.
func (s *Server) GetRootPath() string {
	return s.rootPath
}

// CapturePcap can be called by the application code compiled with tsnet to save a pcap
// of packets which the netstack within tsnet sees. This is expected to be useful during
// debugging, probably not useful for production.
//
// Packets will be written to the pcap until the process exits. The pcap needs a Lua dissector
// to be installed in WireShark in order to decode properly: wgengine/capture/ts-dissector.lua
// in this repository.
// https://tailscale.com/kb/1023/troubleshooting/#can-i-examine-network-traffic-inside-the-encrypted-tunnel
func (s *Server) CapturePcap(ctx context.Context, pcapFile string) error {
	stream, err := s.localClient.StreamDebugCapture(ctx)
	if err != nil {
		return err
	}

	f, err := os.OpenFile(pcapFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		stream.Close()
		return err
	}

	go func(stream io.ReadCloser, f *os.File) {
		defer stream.Close()
		defer f.Close()
		_, _ = io.Copy(f, stream)
	}(stream, f)

	return nil
}

// Sys returns a handle to the Tailscale subsystems of this node.
//
// This is not a stable API, nor are the APIs of the returned subsystems.
func (s *Server) Sys() *tsd.System {
	return s.sys
}

type listenKey struct {
	network string
	host    netip.Addr // or zero value for unspecified
	port    uint16
	funnel  bool
}

type listener struct {
	s       *Server
	keys    []listenKey
	addr    string
	conn    chan net.Conn // unbuffered, never closed
	closedc chan struct{} // closed on [listener.Close]
	closed  bool          // guarded by s.mu

	// gonetLn, if set, is the gonet.Listener that handles new connections.
	// gonetLn is set by [listen] when a TUN is in use and terminates the listener.
	// gonetLn is nil when TUN is nil.
	gonetLn net.Listener
}

func (ln *listener) Accept() (net.Conn, error) {
	if ln.gonetLn != nil {
		return ln.gonetLn.Accept()
	}
	select {
	case c := <-ln.conn:
		return c, nil
	case <-ln.closedc:
		return nil, fmt.Errorf("tsnet: %w", net.ErrClosed)
	}
}

func (ln *listener) Addr() net.Addr {
	if ln.gonetLn != nil {
		return ln.gonetLn.Addr()
	}
	return addr{
		network: ln.keys[0].network,
		addr:    ln.addr,
	}
}

func (ln *listener) Close() error {
	ln.s.mu.Lock()
	defer ln.s.mu.Unlock()
	return ln.closeLocked()
}

// closeLocked closes the listener.
// It must be called with ln.s.mu held.
func (ln *listener) closeLocked() error {
	if ln.closed {
		return fmt.Errorf("tsnet: %w", net.ErrClosed)
	}
	for _, key := range ln.keys {
		if v, ok := ln.s.listeners[key]; ok && v == ln {
			delete(ln.s.listeners, key)
		}
	}
	close(ln.closedc)
	ln.closed = true
	if ln.gonetLn != nil {
		ln.gonetLn.Close()
	}
	return nil
}

func (ln *listener) handle(c net.Conn) {
	select {
	case ln.conn <- c:
		return
	case <-ln.closedc:
	case <-ln.s.shutdownCtx.Done():
	case <-time.After(time.Second):
		// TODO(bradfitz): this isn't ideal. Think about how
		// we how we want to do pushback.
	}
	c.Close()
}

// Server returns the tsnet Server associated with the listener.
func (ln *listener) Server() *Server { return ln.s }

type addr struct {
	network, addr string
}

func (a addr) Network() string { return a.network }
func (a addr) String() string  { return a.addr }

// cleanupListener wraps a net.Listener with a function to be run on Close.
type cleanupListener struct {
	net.Listener
	cleanupOnce sync.Once
	cleanup     func() error // nil if unused
}

func (cl *cleanupListener) Close() error {
	var cleanupErr error
	cl.cleanupOnce.Do(func() {
		if cl.cleanup != nil {
			cleanupErr = cl.cleanup()
		}
	})
	return errors.Join(cl.Listener.Close(), cleanupErr)
}