tstest/natlab/vmtest: add macOS VM support and TestMacOSAndLinuxCanPing

Add macOS VM support to the vmtest integration test framework using
tailmac (Apple Virtualization.framework). The new TestMacOSAndLinuxCanPing
creates a LAN with a Gokrazy arm64 Linux VM (QEMU/HVF) and a macOS VM
(tailmac) and verifies they can ping each other over vnet.

Key changes:

vmtest framework (tstest/natlab/vmtest/):
- Add MacOS OSImage type with IsMacOS field
- Add NoAgent node option for VMs without TTA
- Add LANPing method using TTA's /ping endpoint with retries
- Add --macos-vm-id flag for specifying the base macOS VM
- Set up Unix datagram socket for macOS VMs (ProtocolUnixDGRAM)
- Add tailmac.go for macOS VM clone/configure/launch lifecycle
- Support arm64 QEMU with HVF on macOS hosts (virt machine type)
- Use /tmp for socket paths to avoid 104-byte sun_path limit

tailmac (tstest/tailmac/):
- Add --headless flag to Host.app for GUI-less VM operation
- Use RunLoop.main.run() instead of dispatchMain() (VZ framework
  requires the RunLoop for start/restore callbacks)
- Single-NIC mode in headless (matches llmacstation VM config)
- Add socketpair relay between VZ and the vnet dgram socket
- Fix dispatchMain() bug in tailmac CLI's create command too

gokrazy arm64:
- Add natlab-arm64 Makefile target
- Add gokrazydeps.go for github.com/gokrazy/kernel.arm64
- Add kernel.arm64 dependency to go.mod

The test requires: macOS arm64 host, qemu-system-aarch64, a pre-built
macOS VM (--macos-vm-id flag), and tailmac Host.app built.
This commit is contained in:
Brad Fitzpatrick 2026-04-10 06:43:21 -07:00
parent 03c3551ee5
commit ef69d2041b
13 changed files with 525 additions and 69 deletions

1
go.mod
View File

@ -45,6 +45,7 @@ require (
github.com/godbus/dbus/v5 v5.1.1-0.20230522191255-76236955d466
github.com/gokrazy/breakglass v0.0.0-20251229072214-9dbc0478d486
github.com/gokrazy/gokrazy v0.0.0-20260123094004-294c93fa173c
github.com/gokrazy/kernel.arm64 v0.0.0-20260403054012-807489e0272a
github.com/gokrazy/serial-busybox v0.0.0-20250119153030-ac58ba7574e7
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8
github.com/golang/snappy v0.0.4

2
go.sum
View File

@ -492,6 +492,8 @@ github.com/gokrazy/gokrazy v0.0.0-20260123094004-294c93fa173c/go.mod h1:NtMkrFeD
github.com/gokrazy/internal v0.0.0-20200407075822-660ad467b7c9/go.mod h1:LA5TQy7LcvYGQOy75tkrYkFUhbV2nl5qEBP47PSi2JA=
github.com/gokrazy/internal v0.0.0-20251208203110-3c1aa9087c82 h1:4ghNfD9NaZLpFrqQiBF6mPVFeMYXJSky38ubVA4ic2E=
github.com/gokrazy/internal v0.0.0-20251208203110-3c1aa9087c82/go.mod h1:dQY4EMkD4L5ZjYJ0SPtpgYbV7MIUMCxNIXiOfnZ6jP4=
github.com/gokrazy/kernel.arm64 v0.0.0-20260403054012-807489e0272a h1:fa11POmSLo6fkkcqc+RUIyiqGJzBAOHEe/CCHAA/NGc=
github.com/gokrazy/kernel.arm64 v0.0.0-20260403054012-807489e0272a/go.mod h1:WWx72LXHEesuJxbopusRfSoKJQ6ffdwkT0DZditdrLo=
github.com/gokrazy/serial-busybox v0.0.0-20250119153030-ac58ba7574e7 h1:gurTGc4sL7Ik+IKZ29rhGgHNZQTXPtEXLw+aM9E+/HE=
github.com/gokrazy/serial-busybox v0.0.0-20250119153030-ac58ba7574e7/go.mod h1:OYcG5tSb+QrelmUOO4EZVUFcIHyyZb0QDbEbZFUp1TA=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=

View File

@ -11,3 +11,8 @@ qemu: image
natlab:
go run build.go --build --app=natlabapp
qemu-img convert -O qcow2 natlabapp.img natlabapp.qcow2
# For natlab integration tests on macOS arm64:
natlab-arm64:
go run build.go --build --app=natlabapp.arm64
qemu-img convert -O qcow2 natlabapp.arm64.img natlabapp.arm64.qcow2

View File

@ -0,0 +1,16 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
//go:build for_go_mod_tidy_only
package gokrazydeps
import (
_ "github.com/gokrazy/gokrazy"
_ "github.com/gokrazy/gokrazy/cmd/dhcp"
_ "github.com/gokrazy/kernel.arm64"
_ "github.com/gokrazy/serial-busybox"
_ "tailscale.com/cmd/tailscale"
_ "tailscale.com/cmd/tailscaled"
_ "tailscale.com/cmd/tta"
)

View File

@ -26,10 +26,14 @@ type OSImage struct {
SHA256 string // expected SHA256 hash of the image (of the final qcow2, after any decompression)
MemoryMB int // RAM for the VM
IsGokrazy bool // true for gokrazy images (different QEMU setup)
IsMacOS bool // true for macOS images (launched via tailmac, not QEMU)
}
// GOOS returns the Go OS name for this image.
func (img OSImage) GOOS() string {
if img.IsMacOS {
return "darwin"
}
if img.IsGokrazy {
return "linux"
}
@ -41,6 +45,9 @@ func (img OSImage) GOOS() string {
// GOARCH returns the Go architecture name for this image.
func (img OSImage) GOARCH() string {
if img.IsMacOS {
return "arm64"
}
return "amd64"
}
@ -73,6 +80,15 @@ var (
URL: "https://download.freebsd.org/releases/VM-IMAGES/15.0-RELEASE/amd64/Latest/FreeBSD-15.0-RELEASE-amd64-BASIC-CLOUDINIT-ufs.qcow2.xz",
MemoryMB: 1024,
}
// MacOS is a macOS VM launched via tailmac (Apple Virtualization.framework).
// Requires a pre-created macOS VM image (see tstest/tailmac/README.md).
// Only runs on macOS arm64 hosts.
MacOS = OSImage{
Name: "macos",
IsMacOS: true,
MemoryMB: 4096,
}
)
// imageCacheDir returns the directory for cached VM images.

View File

@ -12,12 +12,23 @@ import (
"os/exec"
"path/filepath"
"regexp"
"runtime"
"strconv"
"time"
"tailscale.com/tstest/natlab/vnet"
)
// qemuBinary returns the QEMU system emulator binary name for the current
// host. On macOS arm64, gokrazy arm64 images use qemu-system-aarch64 with HVF.
// On Linux x86_64, the standard qemu-system-x86_64 with KVM is used.
func qemuBinary() string {
if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
return "qemu-system-aarch64"
}
return "qemu-system-x86_64"
}
// startQEMU launches a QEMU process for the given node.
func (e *Env) startQEMU(n *Node) error {
if n.os.IsGokrazy {
@ -27,7 +38,8 @@ func (e *Env) startQEMU(n *Node) error {
}
// startGokrazyQEMU launches a QEMU process for a gokrazy node.
// This follows the same pattern as tstest/integration/nat/nat_test.go.
// On Linux x86_64, this uses microvm with KVM. On macOS arm64, it uses the
// virt machine type with HVF for fast native arm64 VM execution.
func (e *Env) startGokrazyQEMU(n *Node) error {
disk := filepath.Join(e.tempDir, fmt.Sprintf("%s.qcow2", n.name))
if err := createOverlay(e.gokrazyBase, disk); err != nil {
@ -45,28 +57,56 @@ func (e *Env) startGokrazyQEMU(n *Node) error {
logPath := filepath.Join(e.tempDir, n.name+".log")
args := []string{
"-M", "microvm,isa-serial=off",
"-m", fmt.Sprintf("%dM", n.os.MemoryMB),
"-nodefaults", "-no-user-config", "-nographic",
"-kernel", e.gokrazyKernel,
"-append", "console=hvc0 root=PARTUUID=60c24cc1-f3f9-427a-8199-76baa2d60001/PARTNROFF=1 ro init=/gokrazy/init panic=10 oops=panic pci=off nousb tsc=unstable clocksource=hpet gokrazy.remote_syslog.target=" + sysLogAddr + " tailscale-tta=1" + envBuf.String(),
"-drive", "id=blk0,file=" + disk + ",format=qcow2",
"-device", "virtio-blk-device,drive=blk0",
"-device", "virtio-serial-device",
"-device", "virtio-rng-device",
"-chardev", "file,id=virtiocon0,path=" + logPath,
"-device", "virtconsole,chardev=virtiocon0",
var args []string
if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
// macOS arm64: use virt machine with HVF for native arm64 execution.
args = []string{
"-machine", "virt,gic-version=max",
"-accel", "hvf",
"-cpu", "host",
"-m", fmt.Sprintf("%dM", n.os.MemoryMB),
"-nodefaults", "-no-user-config", "-nographic",
"-kernel", e.gokrazyKernel,
"-append", "console=hvc0 root=PARTUUID=60c24cc1-f3f9-427a-8199-76baa2d60001/PARTNROFF=1 ro init=/gokrazy/init panic=10 oops=panic gokrazy.remote_syslog.target=" + sysLogAddr + " tailscale-tta=1" + envBuf.String(),
"-drive", "id=blk0,file=" + disk + ",format=qcow2,if=none",
"-device", "virtio-blk-pci,drive=blk0",
"-device", "virtio-serial-pci",
"-device", "virtio-rng-pci",
"-chardev", "file,id=virtiocon0,path=" + logPath,
"-device", "virtconsole,chardev=virtiocon0",
}
} else {
// Linux x86_64: use microvm with KVM.
args = []string{
"-M", "microvm,isa-serial=off",
"-m", fmt.Sprintf("%dM", n.os.MemoryMB),
"-nodefaults", "-no-user-config", "-nographic",
"-kernel", e.gokrazyKernel,
"-append", "console=hvc0 root=PARTUUID=60c24cc1-f3f9-427a-8199-76baa2d60001/PARTNROFF=1 ro init=/gokrazy/init panic=10 oops=panic pci=off nousb tsc=unstable clocksource=hpet gokrazy.remote_syslog.target=" + sysLogAddr + " tailscale-tta=1" + envBuf.String(),
"-drive", "id=blk0,file=" + disk + ",format=qcow2",
"-device", "virtio-blk-device,drive=blk0",
"-device", "virtio-serial-device",
"-device", "virtio-rng-device",
"-chardev", "file,id=virtiocon0,path=" + logPath,
"-device", "virtconsole,chardev=virtiocon0",
}
}
// Add network devices — one per NIC.
for i := range n.vnetNode.NumNICs() {
mac := n.vnetNode.NICMac(i)
netdevID := fmt.Sprintf("net%d", i)
args = append(args,
"-netdev", fmt.Sprintf("stream,id=%s,addr.type=unix,addr.path=%s", netdevID, e.sockAddr),
"-device", fmt.Sprintf("virtio-net-device,netdev=%s,mac=%s", netdevID, mac),
)
if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
args = append(args,
"-netdev", fmt.Sprintf("stream,id=%s,addr.type=unix,addr.path=%s", netdevID, e.sockAddr),
"-device", fmt.Sprintf("virtio-net-pci,netdev=%s,mac=%s", netdevID, mac),
)
} else {
args = append(args,
"-netdev", fmt.Sprintf("stream,id=%s,addr.type=unix,addr.path=%s", netdevID, e.sockAddr),
"-device", fmt.Sprintf("virtio-net-device,netdev=%s,mac=%s", netdevID, mac),
)
}
}
return e.launchQEMU(n.name, logPath, args)
@ -91,8 +131,12 @@ func (e *Env) startCloudQEMU(n *Node) error {
logPath := filepath.Join(e.tempDir, n.name+".log")
qmpSock := filepath.Join(e.tempDir, n.name+"-qmp.sock")
accel := "kvm"
if runtime.GOOS == "darwin" {
accel = "hvf"
}
args := []string{
"-machine", "q35,accel=kvm",
"-machine", "q35,accel=" + accel,
"-m", fmt.Sprintf("%dM", n.os.MemoryMB),
"-cpu", "host",
"-smp", "2",
@ -141,7 +185,7 @@ func (e *Env) startCloudQEMU(n *Node) error {
// VM console output goes to logPath (via QEMU's -serial or -chardev).
// QEMU's own stdout/stderr go to logPath.qemu for diagnostics.
func (e *Env) launchQEMU(name, logPath string, args []string) error {
cmd := exec.Command("qemu-system-x86_64", args...)
cmd := exec.Command(qemuBinary(), args...)
// Send stdout/stderr to the log file for any QEMU diagnostic messages.
// Stdin must be /dev/null to prevent QEMU from trying to read.
devNull, err := os.Open(os.DevNull)

View File

@ -0,0 +1,158 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
package vmtest
import (
"encoding/json"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
)
// ensureTailMac locates the pre-built tailmac Host.app binary.
// Users must build it first with "make all" in tstest/tailmac/.
func (e *Env) ensureTailMac() error {
modRoot, err := findModRoot()
if err != nil {
return err
}
e.tailmacDir = filepath.Join(modRoot, "tstest", "tailmac", "bin")
hostApp := filepath.Join(e.tailmacDir, "Host.app", "Contents", "MacOS", "Host")
if _, err := os.Stat(hostApp); err != nil {
return fmt.Errorf("tailmac Host.app not found at %s; run 'make all' in tstest/tailmac/", hostApp)
}
return nil
}
// tailmacConfig is the JSON config format for tailmac VMs.
// It uses the field names from the llmacstation Config type (vmName, mac, etc.)
// since the base VM images are created by llmacstation.
type tailmacConfig struct {
VMName string `json:"vmName"`
MemorySize uint64 `json:"memorySize"`
DiskSize int64 `json:"diskSize"`
Mac string `json:"mac"`
Hostname string `json:"hostname"`
}
// startTailMacVM clones the base macOS VM, configures it for this test's
// vnet, and launches it headlessly via the tailmac Host.app.
//
// The base VM must have been created by llmacstation with a single-NIC config.
// The headless Host.app matches this by using only the socket-based NIC,
// connecting it directly to vnet's dgram socket.
func (e *Env) startTailMacVM(n *Node) error {
baseID := *macosVMID
testID := fmt.Sprintf("vmtest-%s-%d", n.name, os.Getpid())
// Clone the base VM (APFS CoW makes this nearly instant).
home, err := os.UserHomeDir()
if err != nil {
return fmt.Errorf("getting home dir: %w", err)
}
baseDir := filepath.Join(home, "VM.bundle", baseID)
if _, err := os.Stat(baseDir); err != nil {
return fmt.Errorf("base macOS VM %q not found at %s; create with 'llmacstation create --name %s'", baseID, baseDir, baseID)
}
cloneDir := filepath.Join(home, "VM.bundle", testID)
e.t.Logf("[%s] cloning macOS VM %s -> %s", n.name, baseID, testID)
if out, err := exec.Command("cp", "-c", "-r", baseDir, cloneDir).CombinedOutput(); err != nil {
if out2, err2 := exec.Command("cp", "-r", baseDir, cloneDir).CombinedOutput(); err2 != nil {
return fmt.Errorf("cloning macOS VM: %v: %s (APFS clone: %v: %s)", err2, out2, err, out)
}
}
e.t.Cleanup(func() {
os.RemoveAll(cloneDir)
})
// Write config.json with test-specific MAC and the vnet dgram socket path.
// The serverSocket field tells the Swift code where to connect the VM's NIC.
mac := n.vnetNode.NICMac(0)
cfg := struct {
VMName string `json:"vmName"`
MemorySize uint64 `json:"memorySize"`
DiskSize int64 `json:"diskSize"`
Mac string `json:"mac"`
Hostname string `json:"hostname"`
ServerSocket string `json:"serverSocket"`
}{
VMName: testID,
MemorySize: 8 * 1024 * 1024 * 1024, // 8GB to match base VM
DiskSize: 72 * 1024 * 1024 * 1024,
Mac: mac.String(),
Hostname: testID,
ServerSocket: e.dgramSockAddr,
}
cfgData, _ := json.MarshalIndent(cfg, "", " ")
cfgPath := filepath.Join(cloneDir, "config.json")
if err := os.WriteFile(cfgPath, cfgData, 0644); err != nil {
return fmt.Errorf("writing config.json: %w", err)
}
e.t.Logf("[%s] macOS VM config: mac=%s, socket=%s", n.name, mac, e.dgramSockAddr)
// Launch Host.app in headless mode. Headless mode uses a single NIC
// (matching the llmacstation VM config) connected to the dgram socket.
hostBin := filepath.Join(e.tailmacDir, "Host.app", "Contents", "MacOS", "Host")
args := []string{"run", "--id", testID, "--headless"}
logPath := filepath.Join(e.tempDir, n.name+"-tailmac.log")
logFile, err := os.Create(logPath)
if err != nil {
return fmt.Errorf("creating log file: %w", err)
}
cmd := exec.Command(hostBin, args...)
cmd.Stdout = logFile
cmd.Stderr = logFile
devNull, err := os.Open(os.DevNull)
if err != nil {
logFile.Close()
return fmt.Errorf("open /dev/null: %w", err)
}
cmd.Stdin = devNull
if err := cmd.Start(); err != nil {
devNull.Close()
logFile.Close()
return fmt.Errorf("starting tailmac for %s: %w", n.name, err)
}
e.t.Logf("[%s] launched tailmac (pid %d), log: %s", n.name, cmd.Process.Pid, logPath)
// The Swift code creates a client dgram socket at /tmp/qemu-dgram-<id>.sock
clientSock := fmt.Sprintf("/tmp/qemu-dgram-%s.sock", testID)
e.t.Cleanup(func() {
cmd.Process.Signal(os.Interrupt)
done := make(chan error, 1)
go func() { done <- cmd.Wait() }()
select {
case <-done:
case <-time.After(15 * time.Second):
cmd.Process.Kill()
<-done
}
devNull.Close()
logFile.Close()
os.Remove(clientSock)
if e.t.Failed() {
if data, err := os.ReadFile(logPath); err == nil {
lines := strings.Split(string(data), "\n")
start := 0
if len(lines) > 50 {
start = len(lines) - 50
}
e.t.Logf("=== last 50 lines of %s tailmac log ===", n.name)
for _, line := range lines[start:] {
e.t.Logf("[%s] %s", n.name, line)
}
}
}
})
return nil
}

View File

@ -2,13 +2,18 @@
// SPDX-License-Identifier: BSD-3-Clause
// Package vmtest provides a high-level framework for running integration tests
// across multiple QEMU virtual machines connected by natlab's vnet virtual
// network infrastructure. It supports mixed OS types (gokrazy, Ubuntu, Debian)
// and multi-NIC configurations for scenarios like subnet routing.
// across multiple virtual machines connected by natlab's vnet virtual network
// infrastructure. It supports mixed OS types (gokrazy, Ubuntu, Debian, FreeBSD,
// macOS) and multi-NIC configurations for scenarios like subnet routing.
//
// QEMU VMs (Linux, FreeBSD) use stream Unix sockets to connect to vnet.
// macOS VMs use tailmac (Apple Virtualization.framework) with datagram Unix sockets.
//
// Prerequisites:
// - qemu-system-x86_64 and KVM access (typically the "kvm" group; no root required)
// - A built gokrazy natlabapp image (auto-built on first run via "make natlab" in gokrazy/)
// - For QEMU VMs: qemu-system-x86_64 and KVM access (Linux), or qemu-system-aarch64 (macOS arm64)
// - For gokrazy: a built natlabapp image (auto-built on first run via "make natlab" in gokrazy/)
// - For macOS VMs: macOS arm64 host, tailmac built ("make all" in tstest/tailmac/),
// and a pre-created macOS VM (--macos-vm-id flag)
//
// Run tests with:
//
@ -26,6 +31,7 @@ import (
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
@ -41,6 +47,7 @@ import (
var (
runVMTests = flag.Bool("run-vm-tests", false, "run tests that require VMs with KVM")
verboseVMDebug = flag.Bool("verbose-vm-debug", false, "enable verbose debug logging for VM tests")
macosVMID = flag.String("macos-vm-id", "", "base macOS VM identifier in ~/VM.bundle/ for macOS VM tests")
)
// Env is a test environment that manages virtual networks and QEMU VMs.
@ -52,8 +59,10 @@ type Env struct {
nodes []*Node
tempDir string
sockAddr string // shared Unix socket path for all QEMU netdevs
binDir string // directory for compiled binaries
sockAddr string // shared Unix socket path for all QEMU netdevs
dgramSockAddr string // Unix datagram socket path for macOS VMs (tailmac)
binDir string // directory for compiled binaries
tailmacDir string // path to tailmac binaries (Host.app, etc.)
// gokrazy-specific paths
gokrazyBase string // path to gokrazy base qcow2 image
@ -100,6 +109,7 @@ type Node struct {
vnetNode *vnet.Node // primary vnet node (set during Start)
agent *vnet.NodeAgentClient
joinTailnet bool
noAgent bool // true to skip TTA agent setup (e.g. macOS VMs without TTA)
advertiseRoutes string
webServerPort int
sshPort int // host port for SSH debug access (cloud VMs only)
@ -128,6 +138,10 @@ func (e *Env) AddNode(name string, opts ...any) *Node {
case nodeOptNoTailscale:
n.joinTailnet = false
vnetOpts = append(vnetOpts, vnet.DontJoinTailnet)
case nodeOptNoAgent:
n.noAgent = true
n.joinTailnet = false
vnetOpts = append(vnetOpts, vnet.DontJoinTailnet)
case nodeOptAdvertiseRoutes:
n.advertiseRoutes = string(o)
case nodeOptWebServer:
@ -153,6 +167,7 @@ func (n *Node) LanIP(net *vnet.Network) netip.Addr {
type nodeOptOS OSImage
type nodeOptNoTailscale struct{}
type nodeOptNoAgent struct{}
type nodeOptAdvertiseRoutes string
type nodeOptWebServer int
@ -162,6 +177,10 @@ func OS(img OSImage) nodeOptOS { return nodeOptOS(img) }
// DontJoinTailnet returns a NodeOption that prevents the node from running tailscale up.
func DontJoinTailnet() nodeOptNoTailscale { return nodeOptNoTailscale{} }
// NoAgent returns a NodeOption that skips the TTA agent setup and wait for a node.
// Use this for VMs that don't run TTA (e.g. macOS VMs that only need to respond to ICMP).
func NoAgent() nodeOptNoAgent { return nodeOptNoAgent{} }
// AdvertiseRoutes returns a NodeOption that configures the node to advertise
// the given routes (comma-separated CIDRs) when joining the tailnet.
func AdvertiseRoutes(routes string) nodeOptAdvertiseRoutes {
@ -184,12 +203,30 @@ func (e *Env) Start() {
t.Fatal(err)
}
// Check if any macOS nodes are present; if so, verify prerequisites.
hasMacOS := false
for _, n := range e.nodes {
if n.os.IsMacOS {
hasMacOS = true
break
}
}
if hasMacOS {
if runtime.GOOS != "darwin" || runtime.GOARCH != "arm64" {
t.Skip("macOS VM tests require macOS arm64 host")
}
if *macosVMID == "" {
t.Skip("macOS VM tests require --macos-vm-id flag")
}
}
// Determine which GOOS/GOARCH pairs need compiled binaries (non-gokrazy
// images). Gokrazy has binaries built-in, so doesn't need compilation.
// images that aren't macOS). Gokrazy has binaries built-in. macOS VMs
// don't use compiled binaries (no TTA inside the VM for now).
type platform struct{ goos, goarch string }
needPlatform := set.Set[platform]{}
for _, n := range e.nodes {
if !n.os.IsGokrazy {
if !n.os.IsGokrazy && !n.os.IsMacOS {
needPlatform.Add(platform{n.os.GOOS(), n.os.GOARCH()})
}
}
@ -208,7 +245,11 @@ func (e *Env) Start() {
continue
}
didOS.Add(n.os.Name)
if n.os.IsGokrazy {
if n.os.IsMacOS {
eg.Go(func() error {
return e.ensureTailMac()
})
} else if n.os.IsGokrazy {
eg.Go(func() error {
return e.ensureGokrazy(egCtx)
})
@ -247,33 +288,73 @@ func (e *Env) Start() {
// not via the cloud-init HTTP VIP, because network-config must be available
// during init-local before systemd-networkd-wait-online blocks.
// Start Unix socket listener.
e.sockAddr = filepath.Join(e.tempDir, "vnet.sock")
srv, err := net.Listen("unix", e.sockAddr)
if err != nil {
t.Fatalf("listen unix: %v", err)
}
t.Cleanup(func() { srv.Close() })
go func() {
for {
c, err := srv.Accept()
if err != nil {
return
}
go e.server.ServeUnixConn(c.(*net.UnixConn), vnet.ProtocolQEMU)
}
}()
// Launch QEMU processes.
// Start Unix stream socket listener for QEMU VMs.
// Use /tmp for the socket path because the test temp directory path can
// exceed the 104-byte sun_path limit for Unix sockets on macOS.
hasQEMU := false
for _, n := range e.nodes {
if err := e.startQEMU(n); err != nil {
t.Fatalf("startQEMU(%s): %v", n.name, err)
if !n.os.IsMacOS {
hasQEMU = true
break
}
}
if hasQEMU {
e.sockAddr = fmt.Sprintf("/tmp/vmtest-qemu-%d.sock", os.Getpid())
t.Cleanup(func() { os.Remove(e.sockAddr) })
srv, err := net.Listen("unix", e.sockAddr)
if err != nil {
t.Fatalf("listen unix: %v", err)
}
t.Cleanup(func() { srv.Close() })
go func() {
for {
c, err := srv.Accept()
if err != nil {
return
}
go e.server.ServeUnixConn(c.(*net.UnixConn), vnet.ProtocolQEMU)
}
}()
}
// Start Unix datagram socket listener for macOS VMs (tailmac).
// Use /tmp for the socket path because the test temp directory path can
// exceed the 104-byte sun_path limit for Unix sockets on macOS.
if hasMacOS {
e.dgramSockAddr = fmt.Sprintf("/tmp/vmtest-dgram-%d.sock", os.Getpid())
t.Cleanup(func() { os.Remove(e.dgramSockAddr) })
dgramAddr, err := net.ResolveUnixAddr("unixgram", e.dgramSockAddr)
if err != nil {
t.Fatalf("resolve dgram addr: %v", err)
}
uc, err := net.ListenUnixgram("unixgram", dgramAddr)
if err != nil {
t.Fatalf("listen unixgram: %v", err)
}
t.Cleanup(func() { uc.Close() })
go e.server.ServeUnixConn(uc, vnet.ProtocolUnixDGRAM)
}
// Launch VM processes.
for _, n := range e.nodes {
if n.os.IsMacOS {
if err := e.startTailMacVM(n); err != nil {
t.Fatalf("startTailMacVM(%s): %v", n.name, err)
}
} else {
if err := e.startQEMU(n); err != nil {
t.Fatalf("startQEMU(%s): %v", n.name, err)
}
}
}
// Set up agent clients and wait for all agents to connect.
// Skip nodes with noAgent (e.g. macOS VMs without TTA).
for _, n := range e.nodes {
if n.noAgent {
continue
}
n.agent = e.server.NodeAgentClient(n.vnetNode)
n.vnetNode.SetClient(n.agent)
}
@ -281,6 +362,9 @@ func (e *Env) Start() {
// Wait for agents, then bring up tailscale.
var agentEg errgroup.Group
for _, n := range e.nodes {
if n.noAgent {
continue
}
agentEg.Go(func() error {
t.Logf("[%s] waiting for agent...", n.name)
st, err := n.agent.Status(ctx)
@ -575,6 +659,41 @@ func (e *Env) HTTPGet(from *Node, targetURL string) string {
return ""
}
// LANPing pings a LAN IP from the given node using TTA's /ping endpoint.
// It retries until the ping succeeds or the timeout expires, which is useful
// when waiting for the target VM to boot and acquire a DHCP lease.
func (e *Env) LANPing(from *Node, targetIP netip.Addr) {
if from.agent == nil {
e.t.Fatalf("LANPing: node %s has no agent (NoAgent set?)", from.name)
}
e.t.Logf("LANPing: %s -> %s", from.name, targetIP)
for attempt := range 90 {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
reqURL := fmt.Sprintf("http://unused/ping?host=%s", targetIP)
req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
if err != nil {
cancel()
e.t.Fatalf("LANPing: %v", err)
}
res, err := from.agent.HTTPClient.Do(req)
cancel()
if err != nil {
e.logVerbosef("LANPing attempt %d: %v", attempt+1, err)
time.Sleep(2 * time.Second)
continue
}
body, _ := io.ReadAll(res.Body)
res.Body.Close()
if res.StatusCode == 200 {
e.t.Logf("LANPing: %s -> %s succeeded on attempt %d", from.name, targetIP, attempt+1)
return
}
e.logVerbosef("LANPing attempt %d: status %d, body: %s", attempt+1, res.StatusCode, string(body))
time.Sleep(2 * time.Second)
}
e.t.Fatalf("LANPing: %s -> %s timed out after all attempts", from.name, targetIP)
}
// ensureGokrazy finds or builds the gokrazy base image and kernel.
func (e *Env) ensureGokrazy(ctx context.Context) error {
if e.gokrazyBase != "" {
@ -586,22 +705,32 @@ func (e *Env) ensureGokrazy(ctx context.Context) error {
return err
}
e.gokrazyBase = filepath.Join(modRoot, "gokrazy/natlabapp.qcow2")
// On macOS arm64, use the arm64 gokrazy image.
app := "natlabapp"
makeTarget := "natlab"
kernelMod := "github.com/tailscale/gokrazy-kernel"
if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
app = "natlabapp.arm64"
makeTarget = "natlab-arm64"
kernelMod = "github.com/gokrazy/kernel.arm64"
}
e.gokrazyBase = filepath.Join(modRoot, "gokrazy/"+app+".qcow2")
if _, err := os.Stat(e.gokrazyBase); err != nil {
if !os.IsNotExist(err) {
return err
}
e.t.Logf("building gokrazy natlab image...")
cmd := exec.CommandContext(ctx, "make", "natlab")
e.t.Logf("building gokrazy %s image...", app)
cmd := exec.CommandContext(ctx, "make", makeTarget)
cmd.Dir = filepath.Join(modRoot, "gokrazy")
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stdout
if err := cmd.Run(); err != nil {
return fmt.Errorf("make natlab: %w", err)
return fmt.Errorf("make %s: %w", makeTarget, err)
}
}
kernel, err := findKernelPath(filepath.Join(modRoot, "go.mod"))
kernel, err := findKernelPath(filepath.Join(modRoot, "go.mod"), kernelMod)
if err != nil {
return fmt.Errorf("finding kernel: %w", err)
}
@ -661,8 +790,10 @@ func findModRoot() (string, error) {
}
// findKernelPath finds the gokrazy kernel vmlinuz path from go.mod.
func findKernelPath(goMod string) (string, error) {
// Import the same logic as nat_test.go.
// kernelMod is the Go module path for the kernel (e.g.
// "github.com/tailscale/gokrazy-kernel" for x86_64 or
// "github.com/gokrazy/kernel.arm64" for arm64).
func findKernelPath(goMod string, kernelMod string) (string, error) {
b, err := os.ReadFile(goMod)
if err != nil {
return "", err
@ -674,15 +805,15 @@ func findKernelPath(goMod string) (string, error) {
}
goModCache := strings.TrimSpace(string(goModCacheB))
// Parse go.mod to find gokrazy-kernel version.
// Parse go.mod to find the kernel module version.
for _, line := range strings.Split(string(b), "\n") {
line = strings.TrimSpace(line)
if strings.HasPrefix(line, "github.com/tailscale/gokrazy-kernel") {
if strings.HasPrefix(line, kernelMod) {
parts := strings.Fields(line)
if len(parts) >= 2 {
return filepath.Join(goModCache, parts[0]+"@"+parts[1], "vmlinuz"), nil
}
}
}
return "", fmt.Errorf("gokrazy-kernel not found in %s", goMod)
return "", fmt.Errorf("kernel module %s not found in %s", kernelMod, goMod)
}

View File

@ -20,6 +20,32 @@ func TestSubnetRouterFreeBSD(t *testing.T) {
testSubnetRouterForOS(t, vmtest.FreeBSD150)
}
// TestMacOSAndLinuxCanPing verifies basic LAN connectivity between a macOS VM
// (via tailmac/Virtualization.framework) and a Gokrazy Linux VM (via QEMU).
// Neither VM joins a tailnet; this only tests that the vnet virtual network
// routes Ethernet frames correctly between QEMU (stream) and tailmac (dgram)
// protocol clients. The macOS VM responds to ICMP natively (no TTA needed).
func TestMacOSAndLinuxCanPing(t *testing.T) {
env := vmtest.New(t)
lan := env.AddNetwork("192.168.1.1/24")
linux := env.AddNode("linux", lan,
vmtest.OS(vmtest.Gokrazy),
vmtest.DontJoinTailnet())
macos := env.AddNode("macos", lan,
vmtest.OS(vmtest.MacOS),
vmtest.DontJoinTailnet(),
vmtest.NoAgent())
env.Start()
// Ping from Linux to macOS. This verifies bidirectional LAN connectivity
// since ICMP echo requires a reply. LANPing retries until the macOS VM
// has booted and acquired a DHCP lease from vnet.
env.LANPing(linux, macos.LanIP(lan))
}
func testSubnetRouterForOS(t testing.TB, srOS vmtest.OSImage) {
t.Helper()
env := vmtest.New(t)

View File

@ -83,7 +83,7 @@ struct TailMacConfigHelper {
// Outbound network packets
let serverSocket = config.serverSocket
// Inbound network packets
// Inbound network packets bind a client socket so the server can reply.
let clientSockId = config.vmID
let clientSocket = "/tmp/qemu-dgram-\(clientSockId).sock"
@ -118,19 +118,49 @@ struct TailMacConfigHelper {
socklen_t(MemoryLayout<sockaddr_un>.size))
if connectRes == -1 {
print("Error binding virtual network server socket - \(String(cString: strerror(errno)))")
print("Error connecting to server socket \(serverSocket) - \(String(cString: strerror(errno)))")
return networkDevice
}
print("Virtual if mac address is \(config.mac)")
print("Client bound to \(clientSocket)")
print("Connected to server at \(serverSocket)")
print("Socket fd is \(socket)")
// Use a socketpair between VZ and the relay. VZ reads/writes one end;
// background threads relay between the other end and the vnet dgram socket.
// This is more reliable than giving VZ the dgram socket directly.
var spFds: [Int32] = [0, 0]
guard socketpair(AF_UNIX, SOCK_DGRAM, 0, &spFds) == 0 else {
print("socketpair failed: \(String(cString: strerror(errno)))")
return networkDevice
}
let vzFd = spFds[0]
let relayFd = spFds[1]
let handle = FileHandle(fileDescriptor: socket)
let device = VZFileHandleNetworkDeviceAttachment(fileHandle: handle)
let vzHandle = FileHandle(fileDescriptor: vzFd)
let device = VZFileHandleNetworkDeviceAttachment(fileHandle: vzHandle)
networkDevice.attachment = device
// Relay: guestnetwork (read from relayFd, write to dgram socket)
DispatchQueue.global().async {
var buf = [UInt8](repeating: 0, count: 65536)
while true {
let n = Darwin.read(relayFd, &buf, buf.count)
if n <= 0 { break }
Darwin.write(socket, buf, n)
}
}
// Relay: networkguest (read from dgram socket, write to relayFd)
DispatchQueue.global().async {
var buf = [UInt8](repeating: 0, count: 65536)
while true {
let n = Darwin.read(socket, &buf, buf.count)
if n <= 0 { break }
Darwin.write(relayFd, buf, n)
}
}
return networkDevice
}

View File

@ -20,12 +20,31 @@ extension HostCli {
struct Run: ParsableCommand {
@Option var id: String
@Option var share: String?
@Flag(help: "Run without GUI (for automated testing)") var headless: Bool = false
mutating func run() {
config = Config(id)
config.sharedDir = share
print("Running vm with identifier \(id) and sharedDir \(share ?? "<none>")")
_ = NSApplicationMain(CommandLine.argc, CommandLine.unsafeArgv)
if headless {
DispatchQueue.main.async {
let controller = VMController()
controller.createVirtualMachine(headless: true)
let fileManager = FileManager.default
if fileManager.fileExists(atPath: config.saveFileURL.path) {
print("Restoring virtual machine state from \(config.saveFileURL)")
controller.restoreVirtualMachine()
} else {
print("Starting virtual machine")
controller.startVirtualMachine()
}
}
RunLoop.main.run()
} else {
_ = NSApplicationMain(CommandLine.argc, CommandLine.unsafeArgv)
}
}
}
}

View File

@ -81,7 +81,7 @@ class VMController: NSObject, VZVirtualMachineDelegate {
return macPlatform
}
func createVirtualMachine() {
func createVirtualMachine(headless: Bool = false) {
let virtualMachineConfiguration = VZVirtualMachineConfiguration()
virtualMachineConfiguration.platform = createMacPlaform()
@ -90,7 +90,15 @@ class VMController: NSObject, VZVirtualMachineDelegate {
virtualMachineConfiguration.memorySize = helper.computeMemorySize()
virtualMachineConfiguration.graphicsDevices = [helper.createGraphicsDeviceConfiguration()]
virtualMachineConfiguration.storageDevices = [helper.createBlockDeviceConfiguration()]
virtualMachineConfiguration.networkDevices = [helper.createNetworkDeviceConfiguration(), helper.createSocketNetworkDeviceConfiguration()]
if headless {
// In headless mode, use only the socket-based NIC. This matches
// the single-NIC configuration used by llmacstation when creating
// and saving VM state. Using a different NIC count would make
// saved state restoration fail silently.
virtualMachineConfiguration.networkDevices = [helper.createSocketNetworkDeviceConfiguration()]
} else {
virtualMachineConfiguration.networkDevices = [helper.createNetworkDeviceConfiguration(), helper.createSocketNetworkDeviceConfiguration()]
}
virtualMachineConfiguration.pointingDevices = [helper.createPointingDeviceConfiguration()]
virtualMachineConfiguration.keyboards = [helper.createKeyboardConfiguration()]
virtualMachineConfiguration.socketDevices = [helper.createSocketDeviceConfiguration()]

View File

@ -329,7 +329,7 @@ extension Tailmac {
}
}
dispatchMain()
RunLoop.main.run()
}
}
}