From ef69d2041b4741f48b6ca3d92a7c7ce617ea68cc Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Fri, 10 Apr 2026 06:43:21 -0700 Subject: [PATCH] tstest/natlab/vmtest: add macOS VM support and TestMacOSAndLinuxCanPing Add macOS VM support to the vmtest integration test framework using tailmac (Apple Virtualization.framework). The new TestMacOSAndLinuxCanPing creates a LAN with a Gokrazy arm64 Linux VM (QEMU/HVF) and a macOS VM (tailmac) and verifies they can ping each other over vnet. Key changes: vmtest framework (tstest/natlab/vmtest/): - Add MacOS OSImage type with IsMacOS field - Add NoAgent node option for VMs without TTA - Add LANPing method using TTA's /ping endpoint with retries - Add --macos-vm-id flag for specifying the base macOS VM - Set up Unix datagram socket for macOS VMs (ProtocolUnixDGRAM) - Add tailmac.go for macOS VM clone/configure/launch lifecycle - Support arm64 QEMU with HVF on macOS hosts (virt machine type) - Use /tmp for socket paths to avoid 104-byte sun_path limit tailmac (tstest/tailmac/): - Add --headless flag to Host.app for GUI-less VM operation - Use RunLoop.main.run() instead of dispatchMain() (VZ framework requires the RunLoop for start/restore callbacks) - Single-NIC mode in headless (matches llmacstation VM config) - Add socketpair relay between VZ and the vnet dgram socket - Fix dispatchMain() bug in tailmac CLI's create command too gokrazy arm64: - Add natlab-arm64 Makefile target - Add gokrazydeps.go for github.com/gokrazy/kernel.arm64 - Add kernel.arm64 dependency to go.mod The test requires: macOS arm64 host, qemu-system-aarch64, a pre-built macOS VM (--macos-vm-id flag), and tailmac Host.app built. --- go.mod | 1 + go.sum | 2 + gokrazy/Makefile | 5 + gokrazy/natlabapp.arm64/gokrazydeps.go | 16 ++ tstest/natlab/vmtest/images.go | 16 ++ tstest/natlab/vmtest/qemu.go | 82 +++++-- tstest/natlab/vmtest/tailmac.go | 158 +++++++++++++ tstest/natlab/vmtest/vmtest.go | 213 ++++++++++++++---- tstest/natlab/vmtest/vmtest_test.go | 26 +++ .../Swift/Common/TailMacConfigHelper.swift | 40 +++- tstest/tailmac/Swift/Host/HostCli.swift | 21 +- tstest/tailmac/Swift/Host/VMController.swift | 12 +- tstest/tailmac/Swift/TailMac/TailMac.swift | 2 +- 13 files changed, 525 insertions(+), 69 deletions(-) create mode 100644 gokrazy/natlabapp.arm64/gokrazydeps.go create mode 100644 tstest/natlab/vmtest/tailmac.go diff --git a/go.mod b/go.mod index 5c421e167..bdd713a30 100644 --- a/go.mod +++ b/go.mod @@ -45,6 +45,7 @@ require ( github.com/godbus/dbus/v5 v5.1.1-0.20230522191255-76236955d466 github.com/gokrazy/breakglass v0.0.0-20251229072214-9dbc0478d486 github.com/gokrazy/gokrazy v0.0.0-20260123094004-294c93fa173c + github.com/gokrazy/kernel.arm64 v0.0.0-20260403054012-807489e0272a github.com/gokrazy/serial-busybox v0.0.0-20250119153030-ac58ba7574e7 github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 github.com/golang/snappy v0.0.4 diff --git a/go.sum b/go.sum index cc0046798..bde9ebb53 100644 --- a/go.sum +++ b/go.sum @@ -492,6 +492,8 @@ github.com/gokrazy/gokrazy v0.0.0-20260123094004-294c93fa173c/go.mod h1:NtMkrFeD github.com/gokrazy/internal v0.0.0-20200407075822-660ad467b7c9/go.mod h1:LA5TQy7LcvYGQOy75tkrYkFUhbV2nl5qEBP47PSi2JA= github.com/gokrazy/internal v0.0.0-20251208203110-3c1aa9087c82 h1:4ghNfD9NaZLpFrqQiBF6mPVFeMYXJSky38ubVA4ic2E= github.com/gokrazy/internal v0.0.0-20251208203110-3c1aa9087c82/go.mod h1:dQY4EMkD4L5ZjYJ0SPtpgYbV7MIUMCxNIXiOfnZ6jP4= +github.com/gokrazy/kernel.arm64 v0.0.0-20260403054012-807489e0272a h1:fa11POmSLo6fkkcqc+RUIyiqGJzBAOHEe/CCHAA/NGc= +github.com/gokrazy/kernel.arm64 v0.0.0-20260403054012-807489e0272a/go.mod h1:WWx72LXHEesuJxbopusRfSoKJQ6ffdwkT0DZditdrLo= github.com/gokrazy/serial-busybox v0.0.0-20250119153030-ac58ba7574e7 h1:gurTGc4sL7Ik+IKZ29rhGgHNZQTXPtEXLw+aM9E+/HE= github.com/gokrazy/serial-busybox v0.0.0-20250119153030-ac58ba7574e7/go.mod h1:OYcG5tSb+QrelmUOO4EZVUFcIHyyZb0QDbEbZFUp1TA= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= diff --git a/gokrazy/Makefile b/gokrazy/Makefile index bc55f2a52..014866851 100644 --- a/gokrazy/Makefile +++ b/gokrazy/Makefile @@ -11,3 +11,8 @@ qemu: image natlab: go run build.go --build --app=natlabapp qemu-img convert -O qcow2 natlabapp.img natlabapp.qcow2 + +# For natlab integration tests on macOS arm64: +natlab-arm64: + go run build.go --build --app=natlabapp.arm64 + qemu-img convert -O qcow2 natlabapp.arm64.img natlabapp.arm64.qcow2 diff --git a/gokrazy/natlabapp.arm64/gokrazydeps.go b/gokrazy/natlabapp.arm64/gokrazydeps.go new file mode 100644 index 000000000..ea5f029d5 --- /dev/null +++ b/gokrazy/natlabapp.arm64/gokrazydeps.go @@ -0,0 +1,16 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build for_go_mod_tidy_only + +package gokrazydeps + +import ( + _ "github.com/gokrazy/gokrazy" + _ "github.com/gokrazy/gokrazy/cmd/dhcp" + _ "github.com/gokrazy/kernel.arm64" + _ "github.com/gokrazy/serial-busybox" + _ "tailscale.com/cmd/tailscale" + _ "tailscale.com/cmd/tailscaled" + _ "tailscale.com/cmd/tta" +) diff --git a/tstest/natlab/vmtest/images.go b/tstest/natlab/vmtest/images.go index 49eba443f..ce8b2e444 100644 --- a/tstest/natlab/vmtest/images.go +++ b/tstest/natlab/vmtest/images.go @@ -26,10 +26,14 @@ type OSImage struct { SHA256 string // expected SHA256 hash of the image (of the final qcow2, after any decompression) MemoryMB int // RAM for the VM IsGokrazy bool // true for gokrazy images (different QEMU setup) + IsMacOS bool // true for macOS images (launched via tailmac, not QEMU) } // GOOS returns the Go OS name for this image. func (img OSImage) GOOS() string { + if img.IsMacOS { + return "darwin" + } if img.IsGokrazy { return "linux" } @@ -41,6 +45,9 @@ func (img OSImage) GOOS() string { // GOARCH returns the Go architecture name for this image. func (img OSImage) GOARCH() string { + if img.IsMacOS { + return "arm64" + } return "amd64" } @@ -73,6 +80,15 @@ var ( URL: "https://download.freebsd.org/releases/VM-IMAGES/15.0-RELEASE/amd64/Latest/FreeBSD-15.0-RELEASE-amd64-BASIC-CLOUDINIT-ufs.qcow2.xz", MemoryMB: 1024, } + + // MacOS is a macOS VM launched via tailmac (Apple Virtualization.framework). + // Requires a pre-created macOS VM image (see tstest/tailmac/README.md). + // Only runs on macOS arm64 hosts. + MacOS = OSImage{ + Name: "macos", + IsMacOS: true, + MemoryMB: 4096, + } ) // imageCacheDir returns the directory for cached VM images. diff --git a/tstest/natlab/vmtest/qemu.go b/tstest/natlab/vmtest/qemu.go index df56322fa..7c0c5c125 100644 --- a/tstest/natlab/vmtest/qemu.go +++ b/tstest/natlab/vmtest/qemu.go @@ -12,12 +12,23 @@ import ( "os/exec" "path/filepath" "regexp" + "runtime" "strconv" "time" "tailscale.com/tstest/natlab/vnet" ) +// qemuBinary returns the QEMU system emulator binary name for the current +// host. On macOS arm64, gokrazy arm64 images use qemu-system-aarch64 with HVF. +// On Linux x86_64, the standard qemu-system-x86_64 with KVM is used. +func qemuBinary() string { + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + return "qemu-system-aarch64" + } + return "qemu-system-x86_64" +} + // startQEMU launches a QEMU process for the given node. func (e *Env) startQEMU(n *Node) error { if n.os.IsGokrazy { @@ -27,7 +38,8 @@ func (e *Env) startQEMU(n *Node) error { } // startGokrazyQEMU launches a QEMU process for a gokrazy node. -// This follows the same pattern as tstest/integration/nat/nat_test.go. +// On Linux x86_64, this uses microvm with KVM. On macOS arm64, it uses the +// virt machine type with HVF for fast native arm64 VM execution. func (e *Env) startGokrazyQEMU(n *Node) error { disk := filepath.Join(e.tempDir, fmt.Sprintf("%s.qcow2", n.name)) if err := createOverlay(e.gokrazyBase, disk); err != nil { @@ -45,28 +57,56 @@ func (e *Env) startGokrazyQEMU(n *Node) error { logPath := filepath.Join(e.tempDir, n.name+".log") - args := []string{ - "-M", "microvm,isa-serial=off", - "-m", fmt.Sprintf("%dM", n.os.MemoryMB), - "-nodefaults", "-no-user-config", "-nographic", - "-kernel", e.gokrazyKernel, - "-append", "console=hvc0 root=PARTUUID=60c24cc1-f3f9-427a-8199-76baa2d60001/PARTNROFF=1 ro init=/gokrazy/init panic=10 oops=panic pci=off nousb tsc=unstable clocksource=hpet gokrazy.remote_syslog.target=" + sysLogAddr + " tailscale-tta=1" + envBuf.String(), - "-drive", "id=blk0,file=" + disk + ",format=qcow2", - "-device", "virtio-blk-device,drive=blk0", - "-device", "virtio-serial-device", - "-device", "virtio-rng-device", - "-chardev", "file,id=virtiocon0,path=" + logPath, - "-device", "virtconsole,chardev=virtiocon0", + var args []string + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + // macOS arm64: use virt machine with HVF for native arm64 execution. + args = []string{ + "-machine", "virt,gic-version=max", + "-accel", "hvf", + "-cpu", "host", + "-m", fmt.Sprintf("%dM", n.os.MemoryMB), + "-nodefaults", "-no-user-config", "-nographic", + "-kernel", e.gokrazyKernel, + "-append", "console=hvc0 root=PARTUUID=60c24cc1-f3f9-427a-8199-76baa2d60001/PARTNROFF=1 ro init=/gokrazy/init panic=10 oops=panic gokrazy.remote_syslog.target=" + sysLogAddr + " tailscale-tta=1" + envBuf.String(), + "-drive", "id=blk0,file=" + disk + ",format=qcow2,if=none", + "-device", "virtio-blk-pci,drive=blk0", + "-device", "virtio-serial-pci", + "-device", "virtio-rng-pci", + "-chardev", "file,id=virtiocon0,path=" + logPath, + "-device", "virtconsole,chardev=virtiocon0", + } + } else { + // Linux x86_64: use microvm with KVM. + args = []string{ + "-M", "microvm,isa-serial=off", + "-m", fmt.Sprintf("%dM", n.os.MemoryMB), + "-nodefaults", "-no-user-config", "-nographic", + "-kernel", e.gokrazyKernel, + "-append", "console=hvc0 root=PARTUUID=60c24cc1-f3f9-427a-8199-76baa2d60001/PARTNROFF=1 ro init=/gokrazy/init panic=10 oops=panic pci=off nousb tsc=unstable clocksource=hpet gokrazy.remote_syslog.target=" + sysLogAddr + " tailscale-tta=1" + envBuf.String(), + "-drive", "id=blk0,file=" + disk + ",format=qcow2", + "-device", "virtio-blk-device,drive=blk0", + "-device", "virtio-serial-device", + "-device", "virtio-rng-device", + "-chardev", "file,id=virtiocon0,path=" + logPath, + "-device", "virtconsole,chardev=virtiocon0", + } } // Add network devices — one per NIC. for i := range n.vnetNode.NumNICs() { mac := n.vnetNode.NICMac(i) netdevID := fmt.Sprintf("net%d", i) - args = append(args, - "-netdev", fmt.Sprintf("stream,id=%s,addr.type=unix,addr.path=%s", netdevID, e.sockAddr), - "-device", fmt.Sprintf("virtio-net-device,netdev=%s,mac=%s", netdevID, mac), - ) + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + args = append(args, + "-netdev", fmt.Sprintf("stream,id=%s,addr.type=unix,addr.path=%s", netdevID, e.sockAddr), + "-device", fmt.Sprintf("virtio-net-pci,netdev=%s,mac=%s", netdevID, mac), + ) + } else { + args = append(args, + "-netdev", fmt.Sprintf("stream,id=%s,addr.type=unix,addr.path=%s", netdevID, e.sockAddr), + "-device", fmt.Sprintf("virtio-net-device,netdev=%s,mac=%s", netdevID, mac), + ) + } } return e.launchQEMU(n.name, logPath, args) @@ -91,8 +131,12 @@ func (e *Env) startCloudQEMU(n *Node) error { logPath := filepath.Join(e.tempDir, n.name+".log") qmpSock := filepath.Join(e.tempDir, n.name+"-qmp.sock") + accel := "kvm" + if runtime.GOOS == "darwin" { + accel = "hvf" + } args := []string{ - "-machine", "q35,accel=kvm", + "-machine", "q35,accel=" + accel, "-m", fmt.Sprintf("%dM", n.os.MemoryMB), "-cpu", "host", "-smp", "2", @@ -141,7 +185,7 @@ func (e *Env) startCloudQEMU(n *Node) error { // VM console output goes to logPath (via QEMU's -serial or -chardev). // QEMU's own stdout/stderr go to logPath.qemu for diagnostics. func (e *Env) launchQEMU(name, logPath string, args []string) error { - cmd := exec.Command("qemu-system-x86_64", args...) + cmd := exec.Command(qemuBinary(), args...) // Send stdout/stderr to the log file for any QEMU diagnostic messages. // Stdin must be /dev/null to prevent QEMU from trying to read. devNull, err := os.Open(os.DevNull) diff --git a/tstest/natlab/vmtest/tailmac.go b/tstest/natlab/vmtest/tailmac.go new file mode 100644 index 000000000..619290c88 --- /dev/null +++ b/tstest/natlab/vmtest/tailmac.go @@ -0,0 +1,158 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package vmtest + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" +) + +// ensureTailMac locates the pre-built tailmac Host.app binary. +// Users must build it first with "make all" in tstest/tailmac/. +func (e *Env) ensureTailMac() error { + modRoot, err := findModRoot() + if err != nil { + return err + } + e.tailmacDir = filepath.Join(modRoot, "tstest", "tailmac", "bin") + hostApp := filepath.Join(e.tailmacDir, "Host.app", "Contents", "MacOS", "Host") + if _, err := os.Stat(hostApp); err != nil { + return fmt.Errorf("tailmac Host.app not found at %s; run 'make all' in tstest/tailmac/", hostApp) + } + return nil +} + +// tailmacConfig is the JSON config format for tailmac VMs. +// It uses the field names from the llmacstation Config type (vmName, mac, etc.) +// since the base VM images are created by llmacstation. +type tailmacConfig struct { + VMName string `json:"vmName"` + MemorySize uint64 `json:"memorySize"` + DiskSize int64 `json:"diskSize"` + Mac string `json:"mac"` + Hostname string `json:"hostname"` +} + +// startTailMacVM clones the base macOS VM, configures it for this test's +// vnet, and launches it headlessly via the tailmac Host.app. +// +// The base VM must have been created by llmacstation with a single-NIC config. +// The headless Host.app matches this by using only the socket-based NIC, +// connecting it directly to vnet's dgram socket. +func (e *Env) startTailMacVM(n *Node) error { + baseID := *macosVMID + testID := fmt.Sprintf("vmtest-%s-%d", n.name, os.Getpid()) + + // Clone the base VM (APFS CoW makes this nearly instant). + home, err := os.UserHomeDir() + if err != nil { + return fmt.Errorf("getting home dir: %w", err) + } + baseDir := filepath.Join(home, "VM.bundle", baseID) + if _, err := os.Stat(baseDir); err != nil { + return fmt.Errorf("base macOS VM %q not found at %s; create with 'llmacstation create --name %s'", baseID, baseDir, baseID) + } + cloneDir := filepath.Join(home, "VM.bundle", testID) + e.t.Logf("[%s] cloning macOS VM %s -> %s", n.name, baseID, testID) + if out, err := exec.Command("cp", "-c", "-r", baseDir, cloneDir).CombinedOutput(); err != nil { + if out2, err2 := exec.Command("cp", "-r", baseDir, cloneDir).CombinedOutput(); err2 != nil { + return fmt.Errorf("cloning macOS VM: %v: %s (APFS clone: %v: %s)", err2, out2, err, out) + } + } + e.t.Cleanup(func() { + os.RemoveAll(cloneDir) + }) + + // Write config.json with test-specific MAC and the vnet dgram socket path. + // The serverSocket field tells the Swift code where to connect the VM's NIC. + mac := n.vnetNode.NICMac(0) + cfg := struct { + VMName string `json:"vmName"` + MemorySize uint64 `json:"memorySize"` + DiskSize int64 `json:"diskSize"` + Mac string `json:"mac"` + Hostname string `json:"hostname"` + ServerSocket string `json:"serverSocket"` + }{ + VMName: testID, + MemorySize: 8 * 1024 * 1024 * 1024, // 8GB to match base VM + DiskSize: 72 * 1024 * 1024 * 1024, + Mac: mac.String(), + Hostname: testID, + ServerSocket: e.dgramSockAddr, + } + cfgData, _ := json.MarshalIndent(cfg, "", " ") + cfgPath := filepath.Join(cloneDir, "config.json") + if err := os.WriteFile(cfgPath, cfgData, 0644); err != nil { + return fmt.Errorf("writing config.json: %w", err) + } + e.t.Logf("[%s] macOS VM config: mac=%s, socket=%s", n.name, mac, e.dgramSockAddr) + + // Launch Host.app in headless mode. Headless mode uses a single NIC + // (matching the llmacstation VM config) connected to the dgram socket. + hostBin := filepath.Join(e.tailmacDir, "Host.app", "Contents", "MacOS", "Host") + args := []string{"run", "--id", testID, "--headless"} + + logPath := filepath.Join(e.tempDir, n.name+"-tailmac.log") + logFile, err := os.Create(logPath) + if err != nil { + return fmt.Errorf("creating log file: %w", err) + } + + cmd := exec.Command(hostBin, args...) + cmd.Stdout = logFile + cmd.Stderr = logFile + devNull, err := os.Open(os.DevNull) + if err != nil { + logFile.Close() + return fmt.Errorf("open /dev/null: %w", err) + } + cmd.Stdin = devNull + + if err := cmd.Start(); err != nil { + devNull.Close() + logFile.Close() + return fmt.Errorf("starting tailmac for %s: %w", n.name, err) + } + e.t.Logf("[%s] launched tailmac (pid %d), log: %s", n.name, cmd.Process.Pid, logPath) + + // The Swift code creates a client dgram socket at /tmp/qemu-dgram-.sock + clientSock := fmt.Sprintf("/tmp/qemu-dgram-%s.sock", testID) + + e.t.Cleanup(func() { + cmd.Process.Signal(os.Interrupt) + done := make(chan error, 1) + go func() { done <- cmd.Wait() }() + select { + case <-done: + case <-time.After(15 * time.Second): + cmd.Process.Kill() + <-done + } + devNull.Close() + logFile.Close() + os.Remove(clientSock) + + if e.t.Failed() { + if data, err := os.ReadFile(logPath); err == nil { + lines := strings.Split(string(data), "\n") + start := 0 + if len(lines) > 50 { + start = len(lines) - 50 + } + e.t.Logf("=== last 50 lines of %s tailmac log ===", n.name) + for _, line := range lines[start:] { + e.t.Logf("[%s] %s", n.name, line) + } + } + } + }) + + return nil +} diff --git a/tstest/natlab/vmtest/vmtest.go b/tstest/natlab/vmtest/vmtest.go index e6c89467f..5f5320e6d 100644 --- a/tstest/natlab/vmtest/vmtest.go +++ b/tstest/natlab/vmtest/vmtest.go @@ -2,13 +2,18 @@ // SPDX-License-Identifier: BSD-3-Clause // Package vmtest provides a high-level framework for running integration tests -// across multiple QEMU virtual machines connected by natlab's vnet virtual -// network infrastructure. It supports mixed OS types (gokrazy, Ubuntu, Debian) -// and multi-NIC configurations for scenarios like subnet routing. +// across multiple virtual machines connected by natlab's vnet virtual network +// infrastructure. It supports mixed OS types (gokrazy, Ubuntu, Debian, FreeBSD, +// macOS) and multi-NIC configurations for scenarios like subnet routing. +// +// QEMU VMs (Linux, FreeBSD) use stream Unix sockets to connect to vnet. +// macOS VMs use tailmac (Apple Virtualization.framework) with datagram Unix sockets. // // Prerequisites: -// - qemu-system-x86_64 and KVM access (typically the "kvm" group; no root required) -// - A built gokrazy natlabapp image (auto-built on first run via "make natlab" in gokrazy/) +// - For QEMU VMs: qemu-system-x86_64 and KVM access (Linux), or qemu-system-aarch64 (macOS arm64) +// - For gokrazy: a built natlabapp image (auto-built on first run via "make natlab" in gokrazy/) +// - For macOS VMs: macOS arm64 host, tailmac built ("make all" in tstest/tailmac/), +// and a pre-created macOS VM (--macos-vm-id flag) // // Run tests with: // @@ -26,6 +31,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strings" "testing" "time" @@ -41,6 +47,7 @@ import ( var ( runVMTests = flag.Bool("run-vm-tests", false, "run tests that require VMs with KVM") verboseVMDebug = flag.Bool("verbose-vm-debug", false, "enable verbose debug logging for VM tests") + macosVMID = flag.String("macos-vm-id", "", "base macOS VM identifier in ~/VM.bundle/ for macOS VM tests") ) // Env is a test environment that manages virtual networks and QEMU VMs. @@ -52,8 +59,10 @@ type Env struct { nodes []*Node tempDir string - sockAddr string // shared Unix socket path for all QEMU netdevs - binDir string // directory for compiled binaries + sockAddr string // shared Unix socket path for all QEMU netdevs + dgramSockAddr string // Unix datagram socket path for macOS VMs (tailmac) + binDir string // directory for compiled binaries + tailmacDir string // path to tailmac binaries (Host.app, etc.) // gokrazy-specific paths gokrazyBase string // path to gokrazy base qcow2 image @@ -100,6 +109,7 @@ type Node struct { vnetNode *vnet.Node // primary vnet node (set during Start) agent *vnet.NodeAgentClient joinTailnet bool + noAgent bool // true to skip TTA agent setup (e.g. macOS VMs without TTA) advertiseRoutes string webServerPort int sshPort int // host port for SSH debug access (cloud VMs only) @@ -128,6 +138,10 @@ func (e *Env) AddNode(name string, opts ...any) *Node { case nodeOptNoTailscale: n.joinTailnet = false vnetOpts = append(vnetOpts, vnet.DontJoinTailnet) + case nodeOptNoAgent: + n.noAgent = true + n.joinTailnet = false + vnetOpts = append(vnetOpts, vnet.DontJoinTailnet) case nodeOptAdvertiseRoutes: n.advertiseRoutes = string(o) case nodeOptWebServer: @@ -153,6 +167,7 @@ func (n *Node) LanIP(net *vnet.Network) netip.Addr { type nodeOptOS OSImage type nodeOptNoTailscale struct{} +type nodeOptNoAgent struct{} type nodeOptAdvertiseRoutes string type nodeOptWebServer int @@ -162,6 +177,10 @@ func OS(img OSImage) nodeOptOS { return nodeOptOS(img) } // DontJoinTailnet returns a NodeOption that prevents the node from running tailscale up. func DontJoinTailnet() nodeOptNoTailscale { return nodeOptNoTailscale{} } +// NoAgent returns a NodeOption that skips the TTA agent setup and wait for a node. +// Use this for VMs that don't run TTA (e.g. macOS VMs that only need to respond to ICMP). +func NoAgent() nodeOptNoAgent { return nodeOptNoAgent{} } + // AdvertiseRoutes returns a NodeOption that configures the node to advertise // the given routes (comma-separated CIDRs) when joining the tailnet. func AdvertiseRoutes(routes string) nodeOptAdvertiseRoutes { @@ -184,12 +203,30 @@ func (e *Env) Start() { t.Fatal(err) } + // Check if any macOS nodes are present; if so, verify prerequisites. + hasMacOS := false + for _, n := range e.nodes { + if n.os.IsMacOS { + hasMacOS = true + break + } + } + if hasMacOS { + if runtime.GOOS != "darwin" || runtime.GOARCH != "arm64" { + t.Skip("macOS VM tests require macOS arm64 host") + } + if *macosVMID == "" { + t.Skip("macOS VM tests require --macos-vm-id flag") + } + } + // Determine which GOOS/GOARCH pairs need compiled binaries (non-gokrazy - // images). Gokrazy has binaries built-in, so doesn't need compilation. + // images that aren't macOS). Gokrazy has binaries built-in. macOS VMs + // don't use compiled binaries (no TTA inside the VM for now). type platform struct{ goos, goarch string } needPlatform := set.Set[platform]{} for _, n := range e.nodes { - if !n.os.IsGokrazy { + if !n.os.IsGokrazy && !n.os.IsMacOS { needPlatform.Add(platform{n.os.GOOS(), n.os.GOARCH()}) } } @@ -208,7 +245,11 @@ func (e *Env) Start() { continue } didOS.Add(n.os.Name) - if n.os.IsGokrazy { + if n.os.IsMacOS { + eg.Go(func() error { + return e.ensureTailMac() + }) + } else if n.os.IsGokrazy { eg.Go(func() error { return e.ensureGokrazy(egCtx) }) @@ -247,33 +288,73 @@ func (e *Env) Start() { // not via the cloud-init HTTP VIP, because network-config must be available // during init-local before systemd-networkd-wait-online blocks. - // Start Unix socket listener. - e.sockAddr = filepath.Join(e.tempDir, "vnet.sock") - srv, err := net.Listen("unix", e.sockAddr) - if err != nil { - t.Fatalf("listen unix: %v", err) - } - t.Cleanup(func() { srv.Close() }) - - go func() { - for { - c, err := srv.Accept() - if err != nil { - return - } - go e.server.ServeUnixConn(c.(*net.UnixConn), vnet.ProtocolQEMU) - } - }() - - // Launch QEMU processes. + // Start Unix stream socket listener for QEMU VMs. + // Use /tmp for the socket path because the test temp directory path can + // exceed the 104-byte sun_path limit for Unix sockets on macOS. + hasQEMU := false for _, n := range e.nodes { - if err := e.startQEMU(n); err != nil { - t.Fatalf("startQEMU(%s): %v", n.name, err) + if !n.os.IsMacOS { + hasQEMU = true + break + } + } + if hasQEMU { + e.sockAddr = fmt.Sprintf("/tmp/vmtest-qemu-%d.sock", os.Getpid()) + t.Cleanup(func() { os.Remove(e.sockAddr) }) + srv, err := net.Listen("unix", e.sockAddr) + if err != nil { + t.Fatalf("listen unix: %v", err) + } + t.Cleanup(func() { srv.Close() }) + + go func() { + for { + c, err := srv.Accept() + if err != nil { + return + } + go e.server.ServeUnixConn(c.(*net.UnixConn), vnet.ProtocolQEMU) + } + }() + } + + // Start Unix datagram socket listener for macOS VMs (tailmac). + // Use /tmp for the socket path because the test temp directory path can + // exceed the 104-byte sun_path limit for Unix sockets on macOS. + if hasMacOS { + e.dgramSockAddr = fmt.Sprintf("/tmp/vmtest-dgram-%d.sock", os.Getpid()) + t.Cleanup(func() { os.Remove(e.dgramSockAddr) }) + dgramAddr, err := net.ResolveUnixAddr("unixgram", e.dgramSockAddr) + if err != nil { + t.Fatalf("resolve dgram addr: %v", err) + } + uc, err := net.ListenUnixgram("unixgram", dgramAddr) + if err != nil { + t.Fatalf("listen unixgram: %v", err) + } + t.Cleanup(func() { uc.Close() }) + go e.server.ServeUnixConn(uc, vnet.ProtocolUnixDGRAM) + } + + // Launch VM processes. + for _, n := range e.nodes { + if n.os.IsMacOS { + if err := e.startTailMacVM(n); err != nil { + t.Fatalf("startTailMacVM(%s): %v", n.name, err) + } + } else { + if err := e.startQEMU(n); err != nil { + t.Fatalf("startQEMU(%s): %v", n.name, err) + } } } // Set up agent clients and wait for all agents to connect. + // Skip nodes with noAgent (e.g. macOS VMs without TTA). for _, n := range e.nodes { + if n.noAgent { + continue + } n.agent = e.server.NodeAgentClient(n.vnetNode) n.vnetNode.SetClient(n.agent) } @@ -281,6 +362,9 @@ func (e *Env) Start() { // Wait for agents, then bring up tailscale. var agentEg errgroup.Group for _, n := range e.nodes { + if n.noAgent { + continue + } agentEg.Go(func() error { t.Logf("[%s] waiting for agent...", n.name) st, err := n.agent.Status(ctx) @@ -575,6 +659,41 @@ func (e *Env) HTTPGet(from *Node, targetURL string) string { return "" } +// LANPing pings a LAN IP from the given node using TTA's /ping endpoint. +// It retries until the ping succeeds or the timeout expires, which is useful +// when waiting for the target VM to boot and acquire a DHCP lease. +func (e *Env) LANPing(from *Node, targetIP netip.Addr) { + if from.agent == nil { + e.t.Fatalf("LANPing: node %s has no agent (NoAgent set?)", from.name) + } + e.t.Logf("LANPing: %s -> %s", from.name, targetIP) + for attempt := range 90 { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + reqURL := fmt.Sprintf("http://unused/ping?host=%s", targetIP) + req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil) + if err != nil { + cancel() + e.t.Fatalf("LANPing: %v", err) + } + res, err := from.agent.HTTPClient.Do(req) + cancel() + if err != nil { + e.logVerbosef("LANPing attempt %d: %v", attempt+1, err) + time.Sleep(2 * time.Second) + continue + } + body, _ := io.ReadAll(res.Body) + res.Body.Close() + if res.StatusCode == 200 { + e.t.Logf("LANPing: %s -> %s succeeded on attempt %d", from.name, targetIP, attempt+1) + return + } + e.logVerbosef("LANPing attempt %d: status %d, body: %s", attempt+1, res.StatusCode, string(body)) + time.Sleep(2 * time.Second) + } + e.t.Fatalf("LANPing: %s -> %s timed out after all attempts", from.name, targetIP) +} + // ensureGokrazy finds or builds the gokrazy base image and kernel. func (e *Env) ensureGokrazy(ctx context.Context) error { if e.gokrazyBase != "" { @@ -586,22 +705,32 @@ func (e *Env) ensureGokrazy(ctx context.Context) error { return err } - e.gokrazyBase = filepath.Join(modRoot, "gokrazy/natlabapp.qcow2") + // On macOS arm64, use the arm64 gokrazy image. + app := "natlabapp" + makeTarget := "natlab" + kernelMod := "github.com/tailscale/gokrazy-kernel" + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + app = "natlabapp.arm64" + makeTarget = "natlab-arm64" + kernelMod = "github.com/gokrazy/kernel.arm64" + } + + e.gokrazyBase = filepath.Join(modRoot, "gokrazy/"+app+".qcow2") if _, err := os.Stat(e.gokrazyBase); err != nil { if !os.IsNotExist(err) { return err } - e.t.Logf("building gokrazy natlab image...") - cmd := exec.CommandContext(ctx, "make", "natlab") + e.t.Logf("building gokrazy %s image...", app) + cmd := exec.CommandContext(ctx, "make", makeTarget) cmd.Dir = filepath.Join(modRoot, "gokrazy") cmd.Stderr = os.Stderr cmd.Stdout = os.Stdout if err := cmd.Run(); err != nil { - return fmt.Errorf("make natlab: %w", err) + return fmt.Errorf("make %s: %w", makeTarget, err) } } - kernel, err := findKernelPath(filepath.Join(modRoot, "go.mod")) + kernel, err := findKernelPath(filepath.Join(modRoot, "go.mod"), kernelMod) if err != nil { return fmt.Errorf("finding kernel: %w", err) } @@ -661,8 +790,10 @@ func findModRoot() (string, error) { } // findKernelPath finds the gokrazy kernel vmlinuz path from go.mod. -func findKernelPath(goMod string) (string, error) { - // Import the same logic as nat_test.go. +// kernelMod is the Go module path for the kernel (e.g. +// "github.com/tailscale/gokrazy-kernel" for x86_64 or +// "github.com/gokrazy/kernel.arm64" for arm64). +func findKernelPath(goMod string, kernelMod string) (string, error) { b, err := os.ReadFile(goMod) if err != nil { return "", err @@ -674,15 +805,15 @@ func findKernelPath(goMod string) (string, error) { } goModCache := strings.TrimSpace(string(goModCacheB)) - // Parse go.mod to find gokrazy-kernel version. + // Parse go.mod to find the kernel module version. for _, line := range strings.Split(string(b), "\n") { line = strings.TrimSpace(line) - if strings.HasPrefix(line, "github.com/tailscale/gokrazy-kernel") { + if strings.HasPrefix(line, kernelMod) { parts := strings.Fields(line) if len(parts) >= 2 { return filepath.Join(goModCache, parts[0]+"@"+parts[1], "vmlinuz"), nil } } } - return "", fmt.Errorf("gokrazy-kernel not found in %s", goMod) + return "", fmt.Errorf("kernel module %s not found in %s", kernelMod, goMod) } diff --git a/tstest/natlab/vmtest/vmtest_test.go b/tstest/natlab/vmtest/vmtest_test.go index 91c8359f1..e8b6e780f 100644 --- a/tstest/natlab/vmtest/vmtest_test.go +++ b/tstest/natlab/vmtest/vmtest_test.go @@ -20,6 +20,32 @@ func TestSubnetRouterFreeBSD(t *testing.T) { testSubnetRouterForOS(t, vmtest.FreeBSD150) } +// TestMacOSAndLinuxCanPing verifies basic LAN connectivity between a macOS VM +// (via tailmac/Virtualization.framework) and a Gokrazy Linux VM (via QEMU). +// Neither VM joins a tailnet; this only tests that the vnet virtual network +// routes Ethernet frames correctly between QEMU (stream) and tailmac (dgram) +// protocol clients. The macOS VM responds to ICMP natively (no TTA needed). +func TestMacOSAndLinuxCanPing(t *testing.T) { + env := vmtest.New(t) + + lan := env.AddNetwork("192.168.1.1/24") + + linux := env.AddNode("linux", lan, + vmtest.OS(vmtest.Gokrazy), + vmtest.DontJoinTailnet()) + macos := env.AddNode("macos", lan, + vmtest.OS(vmtest.MacOS), + vmtest.DontJoinTailnet(), + vmtest.NoAgent()) + + env.Start() + + // Ping from Linux to macOS. This verifies bidirectional LAN connectivity + // since ICMP echo requires a reply. LANPing retries until the macOS VM + // has booted and acquired a DHCP lease from vnet. + env.LANPing(linux, macos.LanIP(lan)) +} + func testSubnetRouterForOS(t testing.TB, srOS vmtest.OSImage) { t.Helper() env := vmtest.New(t) diff --git a/tstest/tailmac/Swift/Common/TailMacConfigHelper.swift b/tstest/tailmac/Swift/Common/TailMacConfigHelper.swift index fc7f2d89d..796adf1cb 100644 --- a/tstest/tailmac/Swift/Common/TailMacConfigHelper.swift +++ b/tstest/tailmac/Swift/Common/TailMacConfigHelper.swift @@ -83,7 +83,7 @@ struct TailMacConfigHelper { // Outbound network packets let serverSocket = config.serverSocket - // Inbound network packets + // Inbound network packets — bind a client socket so the server can reply. let clientSockId = config.vmID let clientSocket = "/tmp/qemu-dgram-\(clientSockId).sock" @@ -118,19 +118,49 @@ struct TailMacConfigHelper { socklen_t(MemoryLayout.size)) if connectRes == -1 { - print("Error binding virtual network server socket - \(String(cString: strerror(errno)))") + print("Error connecting to server socket \(serverSocket) - \(String(cString: strerror(errno)))") return networkDevice } print("Virtual if mac address is \(config.mac)") print("Client bound to \(clientSocket)") print("Connected to server at \(serverSocket)") - print("Socket fd is \(socket)") + // Use a socketpair between VZ and the relay. VZ reads/writes one end; + // background threads relay between the other end and the vnet dgram socket. + // This is more reliable than giving VZ the dgram socket directly. + var spFds: [Int32] = [0, 0] + guard socketpair(AF_UNIX, SOCK_DGRAM, 0, &spFds) == 0 else { + print("socketpair failed: \(String(cString: strerror(errno)))") + return networkDevice + } + let vzFd = spFds[0] + let relayFd = spFds[1] - let handle = FileHandle(fileDescriptor: socket) - let device = VZFileHandleNetworkDeviceAttachment(fileHandle: handle) + let vzHandle = FileHandle(fileDescriptor: vzFd) + let device = VZFileHandleNetworkDeviceAttachment(fileHandle: vzHandle) networkDevice.attachment = device + + // Relay: guest→network (read from relayFd, write to dgram socket) + DispatchQueue.global().async { + var buf = [UInt8](repeating: 0, count: 65536) + while true { + let n = Darwin.read(relayFd, &buf, buf.count) + if n <= 0 { break } + Darwin.write(socket, buf, n) + } + } + + // Relay: network→guest (read from dgram socket, write to relayFd) + DispatchQueue.global().async { + var buf = [UInt8](repeating: 0, count: 65536) + while true { + let n = Darwin.read(socket, &buf, buf.count) + if n <= 0 { break } + Darwin.write(relayFd, buf, n) + } + } + return networkDevice } diff --git a/tstest/tailmac/Swift/Host/HostCli.swift b/tstest/tailmac/Swift/Host/HostCli.swift index 9c9ae6fa0..177c25172 100644 --- a/tstest/tailmac/Swift/Host/HostCli.swift +++ b/tstest/tailmac/Swift/Host/HostCli.swift @@ -20,12 +20,31 @@ extension HostCli { struct Run: ParsableCommand { @Option var id: String @Option var share: String? + @Flag(help: "Run without GUI (for automated testing)") var headless: Bool = false mutating func run() { config = Config(id) config.sharedDir = share print("Running vm with identifier \(id) and sharedDir \(share ?? "")") - _ = NSApplicationMain(CommandLine.argc, CommandLine.unsafeArgv) + + if headless { + DispatchQueue.main.async { + let controller = VMController() + controller.createVirtualMachine(headless: true) + + let fileManager = FileManager.default + if fileManager.fileExists(atPath: config.saveFileURL.path) { + print("Restoring virtual machine state from \(config.saveFileURL)") + controller.restoreVirtualMachine() + } else { + print("Starting virtual machine") + controller.startVirtualMachine() + } + } + RunLoop.main.run() + } else { + _ = NSApplicationMain(CommandLine.argc, CommandLine.unsafeArgv) + } } } } diff --git a/tstest/tailmac/Swift/Host/VMController.swift b/tstest/tailmac/Swift/Host/VMController.swift index a19d7222e..883bdcdd9 100644 --- a/tstest/tailmac/Swift/Host/VMController.swift +++ b/tstest/tailmac/Swift/Host/VMController.swift @@ -81,7 +81,7 @@ class VMController: NSObject, VZVirtualMachineDelegate { return macPlatform } - func createVirtualMachine() { + func createVirtualMachine(headless: Bool = false) { let virtualMachineConfiguration = VZVirtualMachineConfiguration() virtualMachineConfiguration.platform = createMacPlaform() @@ -90,7 +90,15 @@ class VMController: NSObject, VZVirtualMachineDelegate { virtualMachineConfiguration.memorySize = helper.computeMemorySize() virtualMachineConfiguration.graphicsDevices = [helper.createGraphicsDeviceConfiguration()] virtualMachineConfiguration.storageDevices = [helper.createBlockDeviceConfiguration()] - virtualMachineConfiguration.networkDevices = [helper.createNetworkDeviceConfiguration(), helper.createSocketNetworkDeviceConfiguration()] + if headless { + // In headless mode, use only the socket-based NIC. This matches + // the single-NIC configuration used by llmacstation when creating + // and saving VM state. Using a different NIC count would make + // saved state restoration fail silently. + virtualMachineConfiguration.networkDevices = [helper.createSocketNetworkDeviceConfiguration()] + } else { + virtualMachineConfiguration.networkDevices = [helper.createNetworkDeviceConfiguration(), helper.createSocketNetworkDeviceConfiguration()] + } virtualMachineConfiguration.pointingDevices = [helper.createPointingDeviceConfiguration()] virtualMachineConfiguration.keyboards = [helper.createKeyboardConfiguration()] virtualMachineConfiguration.socketDevices = [helper.createSocketDeviceConfiguration()] diff --git a/tstest/tailmac/Swift/TailMac/TailMac.swift b/tstest/tailmac/Swift/TailMac/TailMac.swift index 3859b9b0b..2271d3bb2 100644 --- a/tstest/tailmac/Swift/TailMac/TailMac.swift +++ b/tstest/tailmac/Swift/TailMac/TailMac.swift @@ -329,7 +329,7 @@ extension Tailmac { } } - dispatchMain() + RunLoop.main.run() } } }