tstest/natlab/vmtest: add --test-version flag

Add a --test-version flag to run the natlab VM tests against
released tailscale/tailscaled binaries downloaded from
pkgs.tailscale.com instead of building from the source tree.

The value can be a concrete release like "1.97.255", or "stable" /
"unstable" which resolve to the latest TarballsVersion on that track
via pkgs.tailscale.com/<track>/?mode=json. The track for a concrete
version is derived from its minor (even=stable, odd=unstable). The
host architecture (amd64 or arm64) selects the tarball.

Tarballs are cached + extracted under
~/.cache/tailscale-vmtest/builds/<version>_<arch>/ so they are not
re-fetched per test. tta is still always built from the local tree.
Cloud VMs (Ubuntu, Debian) pick up the downloaded binaries via the
existing files.tailscale file server. Non-Linux GOOS (FreeBSD) falls
back to building from source since pkgs.tailscale.com only ships
Linux tarballs. Gokrazy nodes continue to use binaries baked into
the gokrazy image; --test-version is a no-op for them.

Updates #13038

Change-Id: I213ef7db362dd17bf69d2685cbf2ab0ec5a3fee1
Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

tstest/natlab/vmtest/version.go

@@ -0,0 +1,195 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vmtest
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"tailscale.com/types/logger"
+)
+
+// versionRE matches a concrete X.Y.Z release version.
+var versionRE = regexp.MustCompile(`^\d+\.\d+\.\d+$`)
+
+// resolveTestVersion returns the concrete release version (e.g. "1.97.255")
+// for the given --test-version flag value. If v is "unstable" or "stable", it
+// queries pkgs.tailscale.com for the latest TarballsVersion on that track.
+// Otherwise it returns v unchanged.
+func resolveTestVersion(ctx context.Context, v string) (string, error) {
+	if v != "unstable" && v != "stable" {
+		if !versionRE.MatchString(v) {
+			return "", fmt.Errorf("invalid --test-version %q: want \"stable\", \"unstable\", or X.Y.Z", v)
+		}
+		return v, nil
+	}
+	url := "https://pkgs.tailscale.com/" + v + "/?mode=json"
+	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+	if err != nil {
+		return "", err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("fetching %s: %w", url, err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return "", fmt.Errorf("fetching %s: HTTP %s", url, resp.Status)
+	}
+	var meta struct {
+		TarballsVersion string
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&meta); err != nil {
+		return "", fmt.Errorf("decoding %s: %w", url, err)
+	}
+	if meta.TarballsVersion == "" {
+		return "", fmt.Errorf("no TarballsVersion in %s response", url)
+	}
+	return meta.TarballsVersion, nil
+}
+
+// versionTrack returns the pkgs.tailscale.com track ("stable" or "unstable")
+// for a release version. Even minors are stable; odd minors are unstable.
+func versionTrack(version string) (string, error) {
+	parts := strings.Split(version, ".")
+	if len(parts) < 2 {
+		return "", fmt.Errorf("bad version %q (expected like 1.97.255)", version)
+	}
+	minor, err := strconv.Atoi(parts[1])
+	if err != nil {
+		return "", fmt.Errorf("bad minor in version %q: %w", version, err)
+	}
+	if minor%2 == 0 {
+		return "stable", nil
+	}
+	return "unstable", nil
+}
+
+// versionCacheRoot returns the root cache directory for downloaded version
+// tarballs.
+func versionCacheRoot() string {
+	if d := os.Getenv("VMTEST_BUILDS_CACHE_DIR"); d != "" {
+		return d
+	}
+	cache, err := os.UserCacheDir()
+	if err != nil {
+		panic(fmt.Sprintf("os.UserCacheDir: %v", err))
+	}
+	return filepath.Join(cache, "tailscale-vmtest", "builds")
+}
+
+// versionCacheDir returns the directory holding the extracted binaries for
+// the given version+arch.
+func versionCacheDir(version, arch string) string {
+	return filepath.Join(versionCacheRoot(), fmt.Sprintf("%s_%s", version, arch))
+}
+
+// ensureVersionBinaries downloads (if needed) and extracts the tailscale
+// release tarball for the given concrete version+arch, returning the
+// directory containing tailscale and tailscaled.
+func ensureVersionBinaries(ctx context.Context, version, arch string, logf logger.Logf) (string, error) {
+	dir := versionCacheDir(version, arch)
+	tailscaled := filepath.Join(dir, "tailscaled")
+	tailscale := filepath.Join(dir, "tailscale")
+	if _, err1 := os.Stat(tailscaled); err1 == nil {
+		if _, err2 := os.Stat(tailscale); err2 == nil {
+			return dir, nil
+		}
+	}
+
+	track, err := versionTrack(version)
+	if err != nil {
+		return "", err
+	}
+	url := fmt.Sprintf("https://pkgs.tailscale.com/%s/tailscale_%s_%s.tgz", track, version, arch)
+	logf("downloading %s", url)
+
+	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+	if err != nil {
+		return "", err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("fetching %s: %w", url, err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return "", fmt.Errorf("fetching %s: HTTP %s", url, resp.Status)
+	}
+
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return "", err
+	}
+
+	gzr, err := gzip.NewReader(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("gzip reader for %s: %w", url, err)
+	}
+	defer gzr.Close()
+	tr := tar.NewReader(gzr)
+
+	wantBase := map[string]bool{
+		"tailscale":  true,
+		"tailscaled": true,
+	}
+	got := map[string]bool{}
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return "", fmt.Errorf("reading tarball %s: %w", url, err)
+		}
+		if hdr.Typeflag != tar.TypeReg {
+			continue
+		}
+		base := path.Base(hdr.Name)
+		if !wantBase[base] {
+			continue
+		}
+		if err := writeAtomic(filepath.Join(dir, base), tr, 0755); err != nil {
+			return "", fmt.Errorf("extracting %s from %s: %w", base, url, err)
+		}
+		got[base] = true
+	}
+	for b := range wantBase {
+		if !got[b] {
+			return "", fmt.Errorf("tarball %s missing %s", url, b)
+		}
+	}
+	logf("extracted %s and %s to %s", "tailscale", "tailscaled", dir)
+	return dir, nil
+}
+
+// writeAtomic writes the contents of r to dst with the given permission
+// bits, by writing to a sibling temp file and renaming on success.
+func writeAtomic(dst string, r io.Reader, perm os.FileMode) error {
+	tmp := dst + ".tmp"
+	f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, perm)
+	if err != nil {
+		return err
+	}
+	if _, err := io.Copy(f, r); err != nil {
+		f.Close()
+		os.Remove(tmp)
+		return err
+	}
+	if err := f.Close(); err != nil {
+		os.Remove(tmp)
+		return err
+	}
+	return os.Rename(tmp, dst)
+}

tstest/natlab/vmtest/version_test.go

@@ -0,0 +1,97 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vmtest
+
+import (
+	"context"
+	"flag"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+var testDownloadVersion = flag.Bool("test-download-version", false, "in TestVersionDownload, actually hit pkgs.tailscale.com")
+
+func TestResolveTestVersionInvalid(t *testing.T) {
+	bad := []string{
+		"",
+		"1.97",
+		"v1.97.255",
+		"1.97.255-pre",
+		"latest",
+		"unstabel",
+	}
+	for _, v := range bad {
+		got, err := resolveTestVersion(context.Background(), v)
+		if err == nil {
+			t.Errorf("resolveTestVersion(%q) = %q, want error", v, got)
+		}
+	}
+}
+
+func TestVersionTrack(t *testing.T) {
+	cases := []struct {
+		v, want string
+	}{
+		{"1.96.4", "stable"},
+		{"1.97.255", "unstable"},
+		{"1.98.0", "stable"},
+	}
+	for _, c := range cases {
+		got, err := versionTrack(c.v)
+		if err != nil {
+			t.Errorf("versionTrack(%q): %v", c.v, err)
+			continue
+		}
+		if got != c.want {
+			t.Errorf("versionTrack(%q) = %q, want %q", c.v, got, c.want)
+		}
+	}
+}
+
+// TestVersionDownload exercises the live network path (download + extract +
+// cache). Skipped by default; set --test-download-version to run.
+func TestVersionDownload(t *testing.T) {
+	if !*testDownloadVersion {
+		t.Skip("set --test-download-version to run")
+	}
+	cacheRoot := t.TempDir()
+	t.Setenv("VMTEST_BUILDS_CACHE_DIR", cacheRoot)
+
+	ctx := context.Background()
+	const version = "1.96.4" // stable
+	dir, err := ensureVersionBinaries(ctx, version, "amd64", t.Logf)
+	if err != nil {
+		t.Fatal(err)
+	}
+	wantDir := filepath.Join(cacheRoot, version+"_amd64")
+	if dir != wantDir {
+		t.Errorf("dir = %q, want %q", dir, wantDir)
+	}
+	for _, name := range []string{"tailscale", "tailscaled"} {
+		fi, err := os.Stat(filepath.Join(dir, name))
+		if err != nil {
+			t.Errorf("missing %s: %v", name, err)
+			continue
+		}
+		if fi.Size() < 1<<20 {
+			t.Errorf("%s suspiciously small: %d bytes", name, fi.Size())
+		}
+	}
+
+	// Re-fetch should be a fast no-op (cache hit).
+	if _, err := ensureVersionBinaries(ctx, version, "amd64", t.Logf); err != nil {
+		t.Fatalf("re-fetch: %v", err)
+	}
+
+	// "unstable" resolution.
+	resolved, err := resolveTestVersion(ctx, "unstable")
+	if err != nil {
+		t.Fatalf("resolveTestVersion(unstable): %v", err)
+	}
+	t.Logf("unstable resolved to %q", resolved)
+	if resolved == "" || resolved == "unstable" {
+		t.Errorf("resolved = %q", resolved)
+	}
+}

tstest/natlab/vmtest/vmtest.go

@@ -42,6 +42,7 @@ import (
 var (
 	runVMTests     = flag.Bool("run-vm-tests", false, "run tests that require VMs with KVM")
 	verboseVMDebug = flag.Bool("verbose-vm-debug", false, "enable verbose debug logging for VM tests")
+	testVersion    = flag.String("test-version", "", `if non-empty, download tailscale & tailscaled at the given release version (e.g. "1.97.255", "unstable", or "stable") instead of building from the source tree`)
 )
 
 // Env is a test environment that manages virtual networks and QEMU VMs.
@@ -56,6 +57,11 @@ type Env struct {
 	sockAddr string // shared Unix socket path for all QEMU netdevs
 	binDir   string // directory for compiled binaries
 
+	// testVersion is the resolved Tailscale release version to use (empty if
+	// building from source). When non-empty, tailscale and tailscaled binaries
+	// are downloaded from pkgs.tailscale.com instead of compiled from the tree.
+	testVersion string
+
 	// gokrazy-specific paths
 	gokrazyBase   string // path to gokrazy base qcow2 image
 	gokrazyKernel string // path to gokrazy kernel
@@ -196,6 +202,17 @@ func (e *Env) Start() {
 		t.Fatal(err)
 	}
 
+	// Resolve --test-version up front (e.g. "unstable" -> "1.97.255") so all
+	// platforms see the same concrete version.
+	if *testVersion != "" {
+		v, err := resolveTestVersion(ctx, *testVersion)
+		if err != nil {
+			t.Fatalf("resolving --test-version=%q: %v", *testVersion, err)
+		}
+		e.testVersion = v
+		t.Logf("using Tailscale release version %s (from --test-version=%q)", v, *testVersion)
+	}
+
 	// Determine which GOOS/GOARCH pairs need compiled binaries (non-gokrazy
 	// images). Gokrazy has binaries built-in, so doesn't need compilation.
 	type platform struct{ goos, goarch string }
@@ -716,8 +733,13 @@ func (e *Env) ensureGokrazy(ctx context.Context) error {
 	return nil
 }
 
-// compileBinariesForOS cross-compiles tta, tailscale, and tailscaled for the
-// given GOOS/GOARCH and places them in e.binDir/<goos>_<goarch>/.
+// compileBinariesForOS prepares the tta, tailscale, and tailscaled binaries
+// for the given GOOS/GOARCH and places them in e.binDir/<goos>_<goarch>/.
+//
+// tta is always built from the local source tree (the test agent must match
+// the test framework). When --test-version is set, tailscale and tailscaled
+// are taken from the downloaded release tarball instead of being compiled
+// from source.
 func (e *Env) compileBinariesForOS(ctx context.Context, goos, goarch string) error {
 	modRoot, err := findModRoot()
 	if err != nil {
@@ -730,14 +752,20 @@ func (e *Env) compileBinariesForOS(ctx context.Context, goos, goarch string) err
 		return err
 	}
 
-	binaries := []struct{ name, pkg string }{
-		{"tta", "./cmd/tta"},
-		{"tailscale", "./cmd/tailscale"},
-		{"tailscaled", "./cmd/tailscaled"},
+	// Use downloaded release binaries only on Linux: pkgs.tailscale.com only
+	// publishes Linux tarballs, so other GOOS values still build from source.
+	useDownloaded := e.testVersion != "" && goos == "linux"
+
+	type binary struct{ name, pkg string }
+	buildBins := []binary{{"tta", "./cmd/tta"}}
+	if !useDownloaded {
+		buildBins = append(buildBins,
+			binary{"tailscale", "./cmd/tailscale"},
+			binary{"tailscaled", "./cmd/tailscaled"})
 	}
 
 	var eg errgroup.Group
-	for _, bin := range binaries {
+	for _, bin := range buildBins {
 		eg.Go(func() error {
 			outPath := filepath.Join(outDir, bin.name)
 			e.t.Logf("compiling %s/%s...", dir, bin.name)
@@ -751,9 +779,36 @@ func (e *Env) compileBinariesForOS(ctx context.Context, goos, goarch string) err
 			return nil
 		})
 	}
+
+	if useDownloaded {
+		eg.Go(func() error {
+			srcDir, err := ensureVersionBinaries(ctx, e.testVersion, goarch, e.t.Logf)
+			if err != nil {
+				return err
+			}
+			for _, name := range []string{"tailscale", "tailscaled"} {
+				if err := copyFile(filepath.Join(srcDir, name), filepath.Join(outDir, name), 0755); err != nil {
+					return fmt.Errorf("staging %s/%s: %w", dir, name, err)
+				}
+			}
+			e.t.Logf("staged version %s tailscale & tailscaled for %s", e.testVersion, dir)
+			return nil
+		})
+	}
+
 	return eg.Wait()
 }
 
+// copyFile copies src to dst with the given permission bits.
+func copyFile(src, dst string, perm os.FileMode) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+	return writeAtomic(dst, in, perm)
+}
+
 // findModRoot returns the root of the Go module (where go.mod is).
 func findModRoot() (string, error) {
 	out, err := exec.Command("go", "env", "GOMOD").CombinedOutput()