package main import ( "bufio" "bytes" "errors" "io" "log" "os" "os/exec" "path/filepath" "slices" "strconv" "strings" "sync" "tailscale.com/cmd/testwrapper/flakytest" "tailscale.com/util/mak" ) func main() { log.SetFlags(log.Lshortfile) log.SetPrefix("testwrapper: ") // Build go args: test [-work] ... var workdir string var args = []string{"test"} if !slices.Contains(args, "-work") && !slices.Contains(args, "--work") { args = append(args, "-work") defer func() { if workdir != "" { // Clean up the WORK directory as the user didn't want it. if err := os.RemoveAll(workdir); err != nil { log.Printf("error removing workdir: %s", err) } } }() } args = append(args, os.Args[1:]...) // Run go test. attempt := 1 r, xerr := run("go", args, []string{attemptenv(attempt)}, os.Stdout, os.Stderr) if nonexecerr(xerr) { log.Fatal("go test: ", xerr) } // Check whether anything needs retried. log.Printf("failures: builds=%d tests=%d retryable=%d", r.buildFailures, r.testFailures, r.testFailuresRetryable) if r.buildFailures > 0 || r.testFailuresRetryable == 0 || r.testFailures > r.testFailuresRetryable { exit(xerr) } // Retry tests we found. const maxAttempts = 3 for cmd := range r.retryCmds { pkg := strings.TrimSuffix(cmdPkg(cmd), ".test") for { attempt++ p := r.retryCmds[cmd] log.Printf("attempt %d: %s %s", attempt, pkg, strings.Join(p.tests, " ")) // Retry the test by invoking the built pkg.test binary directly. pr, xerr := run( cmd, append(p.args, "-test.run=^"+strings.Join(p.tests, "$|^")+"$"), []string{attemptenv(attempt)}, os.Stdout, os.Stdout, // go test copies all underlying pkg.test output to stdout ) if nonexecerr(xerr) { log.Fatalf("%s: %s", cmd, xerr) } if code, _ := exitcode(xerr); code == 0 { break // all tests passed. } if attempt == maxAttempts { log.Fatalf("failed %d times: %s %s", attempt, pkg, strings.Join(p.tests, " ")) } // Try again with the new failure instructions. Hopefully with fewer // failed tests... r.retryCmds[cmd] = pr.retryCmds[cmd] } } } // attemptenv returns the environment variable value K=V used to signal // [flakytest] that it's in a test environment. func attemptenv(attempt int) string { return flakytest.FlakeAttemptEnv + "=" + strconv.Itoa(attempt) } type testRun struct { workDir string buildFailures int testFailures int testFailuresRetryable int retryCmds map[string]pkgRetry // cmd path => retry instructions } type pkgRetry struct { cmd string args []string tests []string } // run executes prog with args and environ, writing output to stdout and stderr // and returns the error from [exec.Cmd.Wait], along with information parsed // from the output about how many builds or tests failed and how to retry them. func run(prog string, args []string, environ []string, stdout, stderr io.Writer) (r testRun, _ error) { cmd := exec.Command(prog, args...) cmd.Env = append(os.Environ(), environ...) cmdout, err := cmd.StdoutPipe() if err != nil { log.Fatalf("StdoutPipe: %s", err) } cmderr, err := cmd.StderrPipe() if err != nil { log.Fatalf("StderrPipe: %s", err) } if err := cmd.Start(); err != nil { log.Fatalf("Start: %s", err) } var wg sync.WaitGroup // Read WORK= from first line of stderr. We retain this so we can clean it // when testwrapper ends. wg.Add(1) go func() { defer wg.Done() err := readthrulines(cmderr, stderr, func(line string) { if r.workDir == "" { if w, ok := strings.CutPrefix(line, "WORK="); ok { r.workDir = w } } }) if err != nil { log.Fatalf("reading stderr: %s", err) } }() wg.Add(1) go func() { defer wg.Done() err := readthrulines(cmdout, stdout, func(line string) { if strings.HasPrefix(line, "--- FAIL: Test") { r.testFailures++ return } if strings.HasPrefix(line, "FAIL\t") && strings.HasSuffix(line, "[build failed]") { r.buildFailures++ return } if _, args, ok := strings.Cut(line, "flakytest: retry:"); ok { wargs := strings.Split(strings.TrimSpace(args), " ") if len(wargs) < 2 { log.Printf("failed to retry log line %q", line) return } test, cmd, args := wargs[0], wargs[1], wargs[2:] p := r.retryCmds[cmd] p.cmd = cmd p.args = args p.tests = append(p.tests, test) mak.Set(&r.retryCmds, cmd, p) r.testFailuresRetryable++ return } }) if err != nil { log.Fatalf("reading stdout: %s", err) } }() wg.Wait() xerr := cmd.Wait() return r, xerr } // exit calls os.Exit with the exit code for err. func exit(err error) { code, _ := exitcode(err) os.Exit(code) } // nonexecerr reports whether err is an error which prevented a program executing. func nonexecerr(err error) bool { if err == nil { return false } xe := &exec.ExitError{} return !errors.As(err, &xe) || xe.ExitCode() < 0 } // exitcode returns a representative error code for err. If err has an // ExitCode() int method, its exit code is returned. func exitcode(err error) (code int, ok bool) { if xe := (interface{ ExitCode() int })(nil); errors.As(err, &xe) { return xe.ExitCode(), true } if err != nil { return 1, false } return 0, false } // readthrulines copies r to w, calling f with each line of text. func readthrulines(r io.Reader, w io.Writer, f func(line string)) error { s := bufio.NewScanner(r) for s.Scan() { line := s.Text() f(line) io.WriteString(w, line) io.WriteString(w, "\n") } return s.Err() } // cmdPkg will return the package of the binary that was built. From Go 1.24 on, // this will return the full package path followed by the ".test" from the // autogenerated main test pkg. For earlier Go versions return base(exe). func cmdPkg(exe string) string { v, _ := exec.Command("go", "version", "-m", exe).Output() _, vp, ok := bytes.Cut(v, []byte("\n\tpath\t")) if ok { p, _, _ := bytes.Cut(vp, []byte("\n")) p = bytes.TrimSpace(p) if len(p) > 0 { return string(p) } } return filepath.Base(exe) }