tstest/build-macos-base-vm: cache IPSW in ~/.cache with freshness checks

The IPSW restore image (~15GB) is now cached in
~/.cache/tailscale/vmtest/macos-ipsw/ and only re-downloaded when
Apple publishes a new version. Freshness is checked via HTTP ETag
headers. At most one IPSW is kept in the cache directory.

The Go program now handles the download itself (with progress
reporting) rather than delegating to the Swift helper, which lets
us do proper HTTP conditional requests. The Swift helper is split
into two modes: "fetch-ipsw-url" (queries VZ framework for the
latest URL) and "install" (installs from a local IPSW file).
This commit is contained in:
Brad Fitzpatrick 2026-04-10 09:02:23 -07:00
parent 647bab9acd
commit a4f623ad4b
2 changed files with 250 additions and 151 deletions

View File

@ -2,73 +2,54 @@
// SPDX-License-Identifier: BSD-3-Clause
// This is a helper program invoked by the Go build-macos-base-vm command.
// It uses Apple's Virtualization.framework to download a macOS IPSW
// restore image and install macOS into a VM disk image.
// It uses Apple's Virtualization.framework for two operations:
//
// Usage: installer <vm-dir> <ipsw-path>
// fetch-ipsw-url prints the URL of the latest supported IPSW
// install <vm-dir> <ipsw> installs macOS from a local IPSW into a VM
import Foundation
import Virtualization
guard CommandLine.arguments.count == 3 else {
fputs("usage: installer <vm-dir> <ipsw-path>\n", stderr)
guard CommandLine.arguments.count >= 2 else {
fputs("usage: installer {fetch-ipsw-url | install <vm-dir> <ipsw-path>}\n", stderr)
exit(1)
}
let vmDir = CommandLine.arguments[1]
let ipswPath = CommandLine.arguments[2]
let mode = CommandLine.arguments[1]
let diskURL = URL(fileURLWithPath: vmDir).appendingPathComponent("Disk.img")
let auxURL = URL(fileURLWithPath: vmDir).appendingPathComponent("AuxiliaryStorage")
let hwModelURL = URL(fileURLWithPath: vmDir).appendingPathComponent("HardwareModel")
let machineIdURL = URL(fileURLWithPath: vmDir).appendingPathComponent("MachineIdentifier")
let diskSize: Int64 = 72 * 1024 * 1024 * 1024 // 72 GB sparse
let memorySize: UInt64 = 8 * 1024 * 1024 * 1024 // 8 GB
// Step 1: Download IPSW if needed.
func downloadIPSW(to path: String, completion: @escaping (URL) -> Void) {
let url = URL(fileURLWithPath: path)
if FileManager.default.fileExists(atPath: path) {
print("Using existing IPSW at \(path)")
completion(url)
return
}
print("Downloading latest macOS restore image...")
switch mode {
case "fetch-ipsw-url":
VZMacOSRestoreImage.fetchLatestSupported { result in
switch result {
case .failure(let error):
fputs("Failed to fetch restore image info: \(error)\n", stderr)
exit(1)
case .success(let image):
print("Downloading from \(image.url)...")
let task = URLSession.shared.downloadTask(with: image.url) { localURL, _, error in
if let error = error {
fputs("Download failed: \(error)\n", stderr)
exit(1)
}
do {
try FileManager.default.moveItem(at: localURL!, to: url)
} catch {
fputs("Failed to move IPSW: \(error)\n", stderr)
exit(1)
}
print("Downloaded IPSW to \(path)")
completion(url)
}
task.progress.observe(\.fractionCompleted, options: [.new]) { _, change in
let pct = Int((change.newValue ?? 0) * 100)
print(" download: \(pct)%")
}
task.resume()
// Print URL to stdout for the Go caller to parse.
print(image.url.absoluteString)
exit(0)
}
}
}
RunLoop.main.run()
// Step 2: Install macOS from IPSW.
func installMacOS(ipswURL: URL) {
print("Loading IPSW...")
VZMacOSRestoreImage.load(from: ipswURL) { result in
case "install":
guard CommandLine.arguments.count == 4 else {
fputs("usage: installer install <vm-dir> <ipsw-path>\n", stderr)
exit(1)
}
let vmDir = CommandLine.arguments[2]
let ipswPath = CommandLine.arguments[3]
let diskURL = URL(fileURLWithPath: vmDir).appendingPathComponent("Disk.img")
let auxURL = URL(fileURLWithPath: vmDir).appendingPathComponent("AuxiliaryStorage")
let hwModelURL = URL(fileURLWithPath: vmDir).appendingPathComponent("HardwareModel")
let machineIdURL = URL(fileURLWithPath: vmDir).appendingPathComponent("MachineIdentifier")
let diskSize: Int64 = 72 * 1024 * 1024 * 1024
let memorySize: UInt64 = 8 * 1024 * 1024 * 1024
fputs("Loading IPSW...\n", stderr)
VZMacOSRestoreImage.load(from: URL(fileURLWithPath: ipswPath)) { result in
switch result {
case .failure(let error):
fputs("Failed to load IPSW: \(error)\n", stderr)
@ -83,82 +64,74 @@ func installMacOS(ipswURL: URL) {
exit(1)
}
DispatchQueue.main.async {
doInstall(restoreImage: restoreImage, macOSConfig: macOSConfig)
// Create disk image (sparse).
let fd = open(diskURL.path, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR)
guard fd != -1 else { fputs("Cannot create disk image.\n", stderr); exit(1) }
guard ftruncate(fd, diskSize) == 0 else { fputs("ftruncate failed.\n", stderr); exit(1) }
close(fd)
// Create platform config.
let platform = VZMacPlatformConfiguration()
platform.auxiliaryStorage = try! VZMacAuxiliaryStorage(
creatingStorageAt: auxURL,
hardwareModel: macOSConfig.hardwareModel,
options: [])
platform.hardwareModel = macOSConfig.hardwareModel
platform.machineIdentifier = VZMacMachineIdentifier()
try! platform.hardwareModel.dataRepresentation.write(to: hwModelURL)
try! platform.machineIdentifier.dataRepresentation.write(to: machineIdURL)
// Build VM config (minimal just enough for installation).
let vmConfig = VZVirtualMachineConfiguration()
vmConfig.platform = platform
vmConfig.bootLoader = VZMacOSBootLoader()
var cpuCount = ProcessInfo.processInfo.processorCount - 1
cpuCount = max(cpuCount, VZVirtualMachineConfiguration.minimumAllowedCPUCount)
cpuCount = min(cpuCount, VZVirtualMachineConfiguration.maximumAllowedCPUCount)
vmConfig.cpuCount = cpuCount
var mem = memorySize
mem = max(mem, VZVirtualMachineConfiguration.minimumAllowedMemorySize)
mem = min(mem, VZVirtualMachineConfiguration.maximumAllowedMemorySize)
vmConfig.memorySize = mem
let gfx = VZMacGraphicsDeviceConfiguration()
gfx.displays = [VZMacGraphicsDisplayConfiguration(widthInPixels: 1920, heightInPixels: 1200, pixelsPerInch: 80)]
vmConfig.graphicsDevices = [gfx]
let disk = try! VZDiskImageStorageDeviceAttachment(url: diskURL, readOnly: false)
vmConfig.storageDevices = [VZVirtioBlockDeviceConfiguration(attachment: disk)]
vmConfig.networkDevices = []
vmConfig.pointingDevices = [VZMacTrackpadConfiguration()]
vmConfig.keyboards = [VZMacKeyboardConfiguration()]
try! vmConfig.validate()
let vm = VZVirtualMachine(configuration: vmConfig)
fputs("Starting macOS installation...\n", stderr)
let installer = VZMacOSInstaller(virtualMachine: vm, restoringFromImageAt: restoreImage.url)
installer.install { result in
switch result {
case .success:
fputs("Installation complete.\n", stderr)
exit(0)
case .failure(let error):
fputs("Installation failed: \(error)\n", stderr)
exit(1)
}
}
_ = installer.progress.observe(\.fractionCompleted, options: [.initial, .new]) { _, change in
let pct = Int((change.newValue ?? 0) * 100)
fputs(" install: \(pct)%\n", stderr)
}
}
}
}
RunLoop.main.run()
default:
fputs("unknown mode: \(mode)\n", stderr)
exit(1)
}
func doInstall(restoreImage: VZMacOSRestoreImage, macOSConfig: VZMacOSConfigurationRequirements) {
// Create disk image.
let fd = open(diskURL.path, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR)
guard fd != -1 else { fputs("Cannot create disk image.\n", stderr); exit(1) }
guard ftruncate(fd, diskSize) == 0 else { fputs("ftruncate failed.\n", stderr); exit(1) }
close(fd)
// Create platform config.
let platform = VZMacPlatformConfiguration()
platform.auxiliaryStorage = try! VZMacAuxiliaryStorage(
creatingStorageAt: auxURL,
hardwareModel: macOSConfig.hardwareModel,
options: [])
platform.hardwareModel = macOSConfig.hardwareModel
platform.machineIdentifier = VZMacMachineIdentifier()
// Save hardware model and machine identifier for future boots.
try! platform.hardwareModel.dataRepresentation.write(to: hwModelURL)
try! platform.machineIdentifier.dataRepresentation.write(to: machineIdURL)
// Build VM config (minimal just enough for installation).
let vmConfig = VZVirtualMachineConfiguration()
vmConfig.platform = platform
vmConfig.bootLoader = VZMacOSBootLoader()
var cpuCount = ProcessInfo.processInfo.processorCount - 1
cpuCount = max(cpuCount, VZVirtualMachineConfiguration.minimumAllowedCPUCount)
cpuCount = min(cpuCount, VZVirtualMachineConfiguration.maximumAllowedCPUCount)
vmConfig.cpuCount = cpuCount
var mem = memorySize
mem = max(mem, VZVirtualMachineConfiguration.minimumAllowedMemorySize)
mem = min(mem, VZVirtualMachineConfiguration.maximumAllowedMemorySize)
vmConfig.memorySize = mem
let gfx = VZMacGraphicsDeviceConfiguration()
gfx.displays = [VZMacGraphicsDisplayConfiguration(widthInPixels: 1920, heightInPixels: 1200, pixelsPerInch: 80)]
vmConfig.graphicsDevices = [gfx]
let disk = try! VZDiskImageStorageDeviceAttachment(url: diskURL, readOnly: false)
vmConfig.storageDevices = [VZVirtioBlockDeviceConfiguration(attachment: disk)]
vmConfig.networkDevices = []
vmConfig.pointingDevices = [VZMacTrackpadConfiguration()]
vmConfig.keyboards = [VZMacKeyboardConfiguration()]
try! vmConfig.validate()
let vm = VZVirtualMachine(configuration: vmConfig)
// Install.
print("Starting macOS installation...")
let installer = VZMacOSInstaller(virtualMachine: vm, restoringFromImageAt: restoreImage.url)
installer.install { result in
switch result {
case .success:
print("Installation complete.")
exit(0)
case .failure(let error):
fputs("Installation failed: \(error)\n", stderr)
exit(1)
}
}
_ = installer.progress.observe(\.fractionCompleted, options: [.initial, .new]) { _, change in
let pct = Int((change.newValue ?? 0) * 100)
print(" install: \(pct)%")
}
}
// Main flow.
downloadIPSW(to: ipswPath) { url in
installMacOS(ipswURL: url)
}
RunLoop.main.run()

View File

@ -10,8 +10,9 @@
//
// go run ./tstest/build-macos-base-vm
//
// The VM is created at ~/.cache/tailscale/vmtest/macos/<name>/ and can be used
// by vmtest tests that include macOS nodes. The IPSW is cached alongside it.
// The VM is created at ~/.cache/tailscale/vmtest/macos/<name>/. The IPSW
// restore image is cached in ~/.cache/tailscale/vmtest/macos-ipsw/ and
// only re-downloaded when Apple publishes a newer version.
//
// This only runs on macOS arm64 (Apple Silicon) and requires the Virtualization
// framework entitlement, so the helper Swift binary is compiled and ad-hoc signed
@ -21,7 +22,9 @@ package main
import (
"flag"
"fmt"
"io"
"log"
"net/http"
"os"
"os/exec"
"path/filepath"
@ -45,9 +48,9 @@ func main() {
if err != nil {
log.Fatal(err)
}
bundleDir := filepath.Join(home, ".cache", "tailscale", "vmtest", "macos")
vmDir := filepath.Join(bundleDir, *vmName)
ipswPath := filepath.Join(bundleDir, "RestoreImage.ipsw")
cacheBase := filepath.Join(home, ".cache", "tailscale", "vmtest")
vmDir := filepath.Join(cacheBase, "macos", *vmName)
ipswDir := filepath.Join(cacheBase, "macos-ipsw")
if _, err := os.Stat(filepath.Join(vmDir, "Disk.img")); err == nil {
if !*rebuild {
@ -60,10 +63,10 @@ func main() {
}
}
os.MkdirAll(bundleDir, 0755)
os.MkdirAll(vmDir, 0755)
os.MkdirAll(ipswDir, 0755)
// Step 1: Build the Swift helper that does the VZ install.
// Step 1: Build the Swift helper.
log.Println("Building macOS VM installer helper...")
helperBin, err := buildSwiftHelper()
if err != nil {
@ -71,29 +74,47 @@ func main() {
}
defer os.RemoveAll(filepath.Dir(helperBin))
// Step 2: Run the helper to download IPSW (if needed) and install macOS.
log.Printf("Installing macOS into %s...", vmDir)
log.Println("(This downloads ~15GB on first run and takes several minutes to install.)")
cmd := exec.Command(helperBin, vmDir, ipswPath)
cmd.Stdout = os.Stdout
// Step 2: Get the latest IPSW URL from Apple via the VZ framework.
log.Println("Checking for latest macOS restore image...")
out, err := exec.Command(helperBin, "fetch-ipsw-url").Output()
if err != nil {
if ee, ok := err.(*exec.ExitError); ok {
log.Fatalf("Fetching IPSW URL: %v\n%s", err, ee.Stderr)
}
log.Fatalf("Fetching IPSW URL: %v", err)
}
ipswURL := strings.TrimSpace(string(out))
log.Printf("Latest IPSW: %s", ipswURL)
// Step 3: Download the IPSW, using the cached copy if unchanged.
ipswPath, err := ensureIPSW(ipswDir, ipswURL)
if err != nil {
log.Fatalf("Downloading IPSW: %v", err)
}
// Step 4: Install macOS from the IPSW.
log.Printf("Installing macOS into %s (this takes a few minutes)...", vmDir)
cmd := exec.Command(helperBin, "install", vmDir, ipswPath)
cmd.Stdout = os.Stderr // Swift helper prints progress to stderr
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
log.Fatalf("macOS installation failed: %v", err)
}
// Step 3: Write config.json.
// Step 5: Write config.json for the tailmac Host.app.
configJSON := fmt.Sprintf(`{
"vmName": %q,
"vmID": %q,
"serverSocket": "/tmp/qemu-dgram.sock",
"memorySize": 8589934592,
"diskSize": 77309411328,
"mac": "52:cc:cc:cc:cc:01",
"hostname": %q
}`, *vmName, *vmName)
"ethermac": "52:cc:cc:cc:ce:01",
"port": 51009
}`, *vmName)
if err := os.WriteFile(filepath.Join(vmDir, "config.json"), []byte(configJSON), 0644); err != nil {
log.Fatalf("Writing config.json: %v", err)
}
// Step 4: Mount the disk and apply post-install fixups.
// Step 6: Mount the disk and apply post-install fixups.
log.Println("Applying post-install fixups (skipping Setup Assistant)...")
if err := applyPostInstallFixups(vmDir); err != nil {
log.Fatalf("Post-install fixups: %v", err)
@ -103,6 +124,120 @@ func main() {
log.Println("Run vmtest tests with: go test ./tstest/natlab/vmtest/ --run-vm-tests -v -run TestMacOS")
}
// ensureIPSW downloads the IPSW to ipswDir if it's not already cached or if
// the remote version has changed. Only one IPSW is kept in the directory.
// Returns the path to the local IPSW file.
func ensureIPSW(ipswDir, ipswURL string) (string, error) {
// Use the filename from the URL (e.g. "UniversalMac_26.4.1_25E253_Restore.ipsw").
urlBase := filepath.Base(ipswURL)
if urlBase == "" || urlBase == "." || urlBase == "/" {
urlBase = "Restore.ipsw"
}
localPath := filepath.Join(ipswDir, urlBase)
// If we already have this exact file, do a conditional GET to check freshness.
if fi, err := os.Stat(localPath); err == nil && fi.Size() > 0 {
fresh, err := checkIPSWFresh(localPath, ipswURL)
if err != nil {
log.Printf("Warning: freshness check failed, using cached IPSW: %v", err)
return localPath, nil
}
if fresh {
log.Printf("Using cached IPSW at %s (%d MB)", localPath, fi.Size()/1024/1024)
return localPath, nil
}
log.Println("Cached IPSW is stale, re-downloading...")
}
// Remove any other .ipsw files in the directory (keep at most one).
entries, _ := os.ReadDir(ipswDir)
for _, e := range entries {
if strings.HasSuffix(e.Name(), ".ipsw") || strings.HasSuffix(e.Name(), ".ipsw.etag") {
os.Remove(filepath.Join(ipswDir, e.Name()))
}
}
log.Printf("Downloading %s (~15GB)...", ipswURL)
tmpPath := localPath + ".tmp"
f, err := os.Create(tmpPath)
if err != nil {
return "", err
}
defer func() {
f.Close()
os.Remove(tmpPath)
}()
resp, err := http.Get(ipswURL)
if err != nil {
return "", fmt.Errorf("HTTP GET: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("HTTP %s", resp.Status)
}
total := resp.ContentLength
pr := &progressReader{r: resp.Body, total: total}
if _, err := io.Copy(f, pr); err != nil {
return "", fmt.Errorf("downloading: %w", err)
}
if err := f.Close(); err != nil {
return "", err
}
if err := os.Rename(tmpPath, localPath); err != nil {
return "", err
}
// Save the ETag for future freshness checks.
if etag := resp.Header.Get("ETag"); etag != "" {
os.WriteFile(localPath+".etag", []byte(etag), 0644)
}
log.Printf("Downloaded IPSW to %s", localPath)
return localPath, nil
}
// checkIPSWFresh does a HEAD request with If-None-Match (ETag) to see if
// the cached IPSW is still current. Returns true if the cache is fresh.
func checkIPSWFresh(localPath, ipswURL string) (bool, error) {
req, err := http.NewRequest("HEAD", ipswURL, nil)
if err != nil {
return false, err
}
if etag, err := os.ReadFile(localPath + ".etag"); err == nil && len(etag) > 0 {
req.Header.Set("If-None-Match", string(etag))
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return false, err
}
resp.Body.Close()
return resp.StatusCode == http.StatusNotModified, nil
}
type progressReader struct {
r io.Reader
total int64
read int64
last int // last printed percent
}
func (pr *progressReader) Read(p []byte) (int, error) {
n, err := pr.r.Read(p)
pr.read += int64(n)
if pr.total > 0 {
pct := int(pr.read * 100 / pr.total)
if pct != pr.last {
pr.last = pct
if pct%5 == 0 {
log.Printf(" download: %d%% (%d / %d MB)", pct, pr.read/1024/1024, pr.total/1024/1024)
}
}
}
return n, err
}
// buildSwiftHelper compiles and signs the embedded Swift installer program.
func buildSwiftHelper() (string, error) {
tmpDir, err := os.MkdirTemp("", "build-macos-vm-*")
@ -110,8 +245,6 @@ func buildSwiftHelper() (string, error) {
return "", err
}
// Find the Swift source file next to this Go file.
// When run via "go run", we need to find it relative to the source.
srcDir, err := findSourceDir()
if err != nil {
return "", fmt.Errorf("finding source dir: %w", err)
@ -128,7 +261,6 @@ func buildSwiftHelper() (string, error) {
return "", fmt.Errorf("swiftc: %v\n%s", err, out)
}
// Sign with the virtualization entitlement.
entPath := filepath.Join(tmpDir, "entitlements.plist")
if err := os.WriteFile(entPath, []byte(entitlementsPlist), 0644); err != nil {
return "", err
@ -143,7 +275,6 @@ func buildSwiftHelper() (string, error) {
}
func findSourceDir() (string, error) {
// Try relative to the working directory first.
candidates := []string{
"tstest/build-macos-base-vm",
".",
@ -153,7 +284,6 @@ func findSourceDir() (string, error) {
return filepath.Abs(c)
}
}
// Try relative to the Go module root.
out, err := exec.Command("go", "env", "GOMOD").CombinedOutput()
if err == nil {
modRoot := filepath.Dir(strings.TrimSpace(string(out)))
@ -170,13 +300,11 @@ func findSourceDir() (string, error) {
func applyPostInstallFixups(vmDir string) error {
diskPath := filepath.Join(vmDir, "Disk.img")
// Attach the disk image without auto-mounting.
out, err := exec.Command("hdiutil", "attach", diskPath, "-nomount").CombinedOutput()
if err != nil {
return fmt.Errorf("hdiutil attach: %v\n%s", err, out)
}
// Parse the top-level disk device from output (e.g. /dev/disk4).
var diskDev string
for _, line := range strings.Split(string(out), "\n") {
fields := strings.Fields(line)
@ -212,7 +340,6 @@ func applyPostInstallFixups(vmDir string) error {
return fmt.Errorf("waiting for APFS Data volume: %w", err)
}
// Mount the Data volume via diskutil (handles APFS permissions correctly).
mountPoint, err := os.MkdirTemp("", "vm-data-*")
if err != nil {
return err
@ -225,7 +352,6 @@ func applyPostInstallFixups(vmDir string) error {
}
defer exec.Command("diskutil", "unmount", mountPoint).Run()
// Create .AppleSetupDone to skip the Setup Assistant.
dbDir := filepath.Join(mountPoint, "private", "var", "db")
if err := os.MkdirAll(dbDir, 0755); err != nil {
return fmt.Errorf("creating var/db: %v", err)