From 641a90ea33b07e4550eb244ad02f6d1b4b30baeb Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Wed, 20 Aug 2025 16:24:00 -0700 Subject: [PATCH] net/sockopts,wgengine/magicsock: export socket buffer sizing logic (#16909) For eventual use by net/udprelay.Server Updates tailscale/corp#31164 Signed-off-by: Jordan Whited --- cmd/k8s-operator/depaware.txt | 1 + cmd/tailscaled/depaware.txt | 1 + cmd/tsidp/depaware.txt | 1 + net/sockopts/sockopts.go | 37 +++++++++++++++++ net/sockopts/sockopts_default.go | 21 ++++++++++ net/sockopts/sockopts_linux.go | 40 +++++++++++++++++++ .../sockopts/sockopts_unix_test.go | 7 ++-- tsnet/depaware.txt | 1 + wgengine/magicsock/magicsock.go | 26 ++++++------ wgengine/magicsock/magicsock_default.go | 7 ---- wgengine/magicsock/magicsock_linux.go | 29 -------------- 11 files changed, 119 insertions(+), 52 deletions(-) create mode 100644 net/sockopts/sockopts.go create mode 100644 net/sockopts/sockopts_default.go create mode 100644 net/sockopts/sockopts_linux.go rename wgengine/magicsock/magicsock_unix_test.go => net/sockopts/sockopts_unix_test.go (87%) diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index 1ecef4953..d9cc43e6b 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -867,6 +867,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/net/portmapper from tailscale.com/ipn/localapi+ tailscale.com/net/proxymux from tailscale.com/tsnet tailscale.com/net/routetable from tailscale.com/doctor/routetable + tailscale.com/net/sockopts from tailscale.com/wgengine/magicsock tailscale.com/net/socks5 from tailscale.com/tsnet tailscale.com/net/sockstats from tailscale.com/control/controlclient+ tailscale.com/net/stun from tailscale.com/ipn/localapi+ diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index e60c1cb04..219de5b0c 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -339,6 +339,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/net/portmapper from tailscale.com/ipn/localapi+ tailscale.com/net/proxymux from tailscale.com/cmd/tailscaled tailscale.com/net/routetable from tailscale.com/doctor/routetable + tailscale.com/net/sockopts from tailscale.com/wgengine/magicsock tailscale.com/net/socks5 from tailscale.com/cmd/tailscaled tailscale.com/net/sockstats from tailscale.com/control/controlclient+ tailscale.com/net/stun from tailscale.com/ipn/localapi+ diff --git a/cmd/tsidp/depaware.txt b/cmd/tsidp/depaware.txt index 5e558a0cd..2cd76f91a 100644 --- a/cmd/tsidp/depaware.txt +++ b/cmd/tsidp/depaware.txt @@ -297,6 +297,7 @@ tailscale.com/cmd/tsidp dependencies: (generated by github.com/tailscale/depawar tailscale.com/net/portmapper from tailscale.com/ipn/localapi+ tailscale.com/net/proxymux from tailscale.com/tsnet tailscale.com/net/routetable from tailscale.com/doctor/routetable + tailscale.com/net/sockopts from tailscale.com/wgengine/magicsock tailscale.com/net/socks5 from tailscale.com/tsnet tailscale.com/net/sockstats from tailscale.com/control/controlclient+ tailscale.com/net/stun from tailscale.com/ipn/localapi+ diff --git a/net/sockopts/sockopts.go b/net/sockopts/sockopts.go new file mode 100644 index 000000000..0c0ee7692 --- /dev/null +++ b/net/sockopts/sockopts.go @@ -0,0 +1,37 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package sockopts contains logic for applying socket options. +package sockopts + +import ( + "net" + "runtime" + + "tailscale.com/types/nettype" +) + +// BufferDirection represents either the read/receive or write/send direction +// of a socket buffer. +type BufferDirection string + +const ( + ReadDirection BufferDirection = "read" + WriteDirection BufferDirection = "write" +) + +func portableSetBufferSize(pconn nettype.PacketConn, direction BufferDirection, size int) error { + if runtime.GOOS == "plan9" { + // Not supported. Don't try. Avoid logspam. + return nil + } + var err error + if c, ok := pconn.(*net.UDPConn); ok { + if direction == WriteDirection { + err = c.SetWriteBuffer(size) + } else { + err = c.SetReadBuffer(size) + } + } + return err +} diff --git a/net/sockopts/sockopts_default.go b/net/sockopts/sockopts_default.go new file mode 100644 index 000000000..3cc8679b5 --- /dev/null +++ b/net/sockopts/sockopts_default.go @@ -0,0 +1,21 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !linux + +package sockopts + +import ( + "tailscale.com/types/nettype" +) + +// SetBufferSize sets pconn's buffer to size for direction. size may be silently +// capped depending on platform. +// +// errForce is only relevant for Linux, and will always be nil otherwise, +// but we maintain a consistent cross-platform API. +// +// If pconn is not a [*net.UDPConn], then SetBufferSize is no-op. +func SetBufferSize(pconn nettype.PacketConn, direction BufferDirection, size int) (errForce error, errPortable error) { + return nil, portableSetBufferSize(pconn, direction, size) +} diff --git a/net/sockopts/sockopts_linux.go b/net/sockopts/sockopts_linux.go new file mode 100644 index 000000000..5d778d380 --- /dev/null +++ b/net/sockopts/sockopts_linux.go @@ -0,0 +1,40 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build linux + +package sockopts + +import ( + "net" + "syscall" + + "tailscale.com/types/nettype" +) + +// SetBufferSize sets pconn's buffer to size for direction. It attempts +// (errForce) to set SO_SNDBUFFORCE or SO_RECVBUFFORCE which can overcome the +// limit of net.core.{r,w}mem_max, but require CAP_NET_ADMIN. It falls back to +// the portable implementation (errPortable) if that fails, which may be +// silently capped to net.core.{r,w}mem_max. +// +// If pconn is not a [*net.UDPConn], then SetBufferSize is no-op. +func SetBufferSize(pconn nettype.PacketConn, direction BufferDirection, size int) (errForce error, errPortable error) { + opt := syscall.SO_RCVBUFFORCE + if direction == WriteDirection { + opt = syscall.SO_SNDBUFFORCE + } + if c, ok := pconn.(*net.UDPConn); ok { + var rc syscall.RawConn + rc, errForce = c.SyscallConn() + if errForce == nil { + rc.Control(func(fd uintptr) { + errForce = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, opt, size) + }) + } + if errForce != nil { + errPortable = portableSetBufferSize(pconn, direction, size) + } + } + return errForce, errPortable +} diff --git a/wgengine/magicsock/magicsock_unix_test.go b/net/sockopts/sockopts_unix_test.go similarity index 87% rename from wgengine/magicsock/magicsock_unix_test.go rename to net/sockopts/sockopts_unix_test.go index b0700a8eb..ebb4354ac 100644 --- a/wgengine/magicsock/magicsock_unix_test.go +++ b/net/sockopts/sockopts_unix_test.go @@ -3,7 +3,7 @@ //go:build unix -package magicsock +package sockopts import ( "net" @@ -13,7 +13,7 @@ import ( "tailscale.com/types/nettype" ) -func TestTrySetSocketBuffer(t *testing.T) { +func TestSetBufferSize(t *testing.T) { c, err := net.ListenPacket("udp", ":0") if err != nil { t.Fatal(err) @@ -42,7 +42,8 @@ func TestTrySetSocketBuffer(t *testing.T) { curRcv, curSnd := getBufs() - trySetSocketBuffer(c.(nettype.PacketConn), t.Logf) + SetBufferSize(c.(nettype.PacketConn), ReadDirection, 7<<20) + SetBufferSize(c.(nettype.PacketConn), WriteDirection, 7<<20) newRcv, newSnd := getBufs() diff --git a/tsnet/depaware.txt b/tsnet/depaware.txt index 9ad340c90..d7d5be658 100644 --- a/tsnet/depaware.txt +++ b/tsnet/depaware.txt @@ -293,6 +293,7 @@ tailscale.com/tsnet dependencies: (generated by github.com/tailscale/depaware) tailscale.com/net/portmapper from tailscale.com/ipn/localapi+ tailscale.com/net/proxymux from tailscale.com/tsnet tailscale.com/net/routetable from tailscale.com/doctor/routetable + tailscale.com/net/sockopts from tailscale.com/wgengine/magicsock tailscale.com/net/socks5 from tailscale.com/tsnet tailscale.com/net/sockstats from tailscale.com/control/controlclient+ tailscale.com/net/stun from tailscale.com/ipn/localapi+ diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index a99a0a8e3..a59a38f65 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -45,6 +45,7 @@ import ( "tailscale.com/net/packet" "tailscale.com/net/ping" "tailscale.com/net/portmapper" + "tailscale.com/net/sockopts" "tailscale.com/net/sockstats" "tailscale.com/net/stun" "tailscale.com/net/tstun" @@ -3857,20 +3858,19 @@ func (c *Conn) DebugForcePreferDERP(n int) { c.netChecker.SetForcePreferredDERP(n) } -// portableTrySetSocketBuffer sets SO_SNDBUF and SO_RECVBUF on pconn to socketBufferSize, -// logging an error if it occurs. -func portableTrySetSocketBuffer(pconn nettype.PacketConn, logf logger.Logf) { - if runtime.GOOS == "plan9" { - // Not supported. Don't try. Avoid logspam. - return - } - if c, ok := pconn.(*net.UDPConn); ok { - // Attempt to increase the buffer size, and allow failures. - if err := c.SetReadBuffer(socketBufferSize); err != nil { - logf("magicsock: failed to set UDP read buffer size to %d: %v", socketBufferSize, err) +// trySetSocketBuffer attempts to set SO_SNDBUFFORCE and SO_RECVBUFFORCE which +// can overcome the limit of net.core.{r,w}mem_max, but require CAP_NET_ADMIN. +// It falls back to the portable implementation if that fails, which may be +// silently capped to net.core.{r,w}mem_max. +func trySetSocketBuffer(pconn nettype.PacketConn, logf logger.Logf) { + directions := []sockopts.BufferDirection{sockopts.ReadDirection, sockopts.WriteDirection} + for _, direction := range directions { + forceErr, portableErr := sockopts.SetBufferSize(pconn, direction, socketBufferSize) + if forceErr != nil { + logf("magicsock: [warning] failed to force-set UDP %v buffer size to %d: %v; using kernel default values (impacts throughput only)", direction, socketBufferSize, forceErr) } - if err := c.SetWriteBuffer(socketBufferSize); err != nil { - logf("magicsock: failed to set UDP write buffer size to %d: %v", socketBufferSize, err) + if portableErr != nil { + logf("magicsock: failed to set UDP %v buffer size to %d: %v", direction, socketBufferSize, portableErr) } } } diff --git a/wgengine/magicsock/magicsock_default.go b/wgengine/magicsock/magicsock_default.go index 4922f2c09..1c315034a 100644 --- a/wgengine/magicsock/magicsock_default.go +++ b/wgengine/magicsock/magicsock_default.go @@ -9,15 +9,8 @@ import ( "errors" "fmt" "io" - - "tailscale.com/types/logger" - "tailscale.com/types/nettype" ) func (c *Conn) listenRawDisco(family string) (io.Closer, error) { return nil, fmt.Errorf("raw disco listening not supported on this OS: %w", errors.ErrUnsupported) } - -func trySetSocketBuffer(pconn nettype.PacketConn, logf logger.Logf) { - portableTrySetSocketBuffer(pconn, logf) -} diff --git a/wgengine/magicsock/magicsock_linux.go b/wgengine/magicsock/magicsock_linux.go index 3369bcb89..cad0e9b5e 100644 --- a/wgengine/magicsock/magicsock_linux.go +++ b/wgengine/magicsock/magicsock_linux.go @@ -13,7 +13,6 @@ import ( "net" "net/netip" "strings" - "syscall" "time" "github.com/mdlayher/socket" @@ -28,7 +27,6 @@ import ( "tailscale.com/types/ipproto" "tailscale.com/types/key" "tailscale.com/types/logger" - "tailscale.com/types/nettype" ) const ( @@ -489,30 +487,3 @@ func printSockaddr(sa unix.Sockaddr) string { return fmt.Sprintf("unknown(%T)", sa) } } - -// trySetSocketBuffer attempts to set SO_SNDBUFFORCE and SO_RECVBUFFORCE which -// can overcome the limit of net.core.{r,w}mem_max, but require CAP_NET_ADMIN. -// It falls back to the portable implementation if that fails, which may be -// silently capped to net.core.{r,w}mem_max. -func trySetSocketBuffer(pconn nettype.PacketConn, logf logger.Logf) { - if c, ok := pconn.(*net.UDPConn); ok { - var errRcv, errSnd error - rc, err := c.SyscallConn() - if err == nil { - rc.Control(func(fd uintptr) { - errRcv = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_RCVBUFFORCE, socketBufferSize) - if errRcv != nil { - logf("magicsock: [warning] failed to force-set UDP read buffer size to %d: %v; using kernel default values (impacts throughput only)", socketBufferSize, errRcv) - } - errSnd = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_SNDBUFFORCE, socketBufferSize) - if errSnd != nil { - logf("magicsock: [warning] failed to force-set UDP write buffer size to %d: %v; using kernel default values (impacts throughput only)", socketBufferSize, errSnd) - } - }) - } - - if err != nil || errRcv != nil || errSnd != nil { - portableTrySetSocketBuffer(pconn, logf) - } - } -}