misc/genreadme: port from corp (#19477)

also port pkgdoc, into the tempfork folder git rev from corp at the time this copy was made: - e909fc93595414c90ff1339cece7c84500ab3c36 Updates #19470 Change-Id: I3d98d82020a2b336647b795210dcb7065dfa44d7 Change-Id: Ie63141860b76dd2d5ae3ff52f8a4bcdf6106421e Signed-off-by: Walter Poupore <walterp@tailscale.com>
2026-05-04 19:56:35 +02:00 · 2026-04-21 12:18:37 -07:00 · 2026-04-21 12:18:37 -07:00 · 04415b8177
commit 04415b8177
parent 1669b0d3d4
2 changed files with 421 additions and 0 deletions
--- a/misc/genreadme/genreadme.go
+++ b/misc/genreadme/genreadme.go
@ -0,0 +1,223 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+// The genreadme tool generates/updates README.md files in the tailscale repo.
+//
+// # Running
+//
+// From the repo root, run: `./tool/go run ./misc/genreadme` and it will update all
+// the README.md files that are stale in the tree.
+package main
+
+import (
+	"bytes"
+	"errors"
+	"flag"
+	"fmt"
+	"go/parser"
+	"go/token"
+	"io"
+	"io/fs"
+	"log"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"github.com/creachadair/taskgroup"
+	"tailscale.com/tempfork/pkgdoc"
+)
+
+var skip = map[string]bool{
+	"out": true,
+}
+
+// bkSkip lists directories where the generated file should not mention
+// Buildkite because a deploy workflow is not set up for them.
+var bkSkip = map[string]bool{}
+
+func main() {
+	flag.Parse()
+	root := "."
+	switch flag.NArg() {
+	case 0:
+	case 1:
+		root = flag.Arg(0)
+		root = strings.TrimPrefix(root, "./")
+		root = strings.TrimSuffix(root, "/")
+	default:
+		log.Fatalf("Usage: genreadme [dir]")
+	}
+
+	var updateErrs []error
+	g, run := taskgroup.New(func(err error) {
+		updateErrs = append(updateErrs, err)
+	}).Limit(runtime.NumCPU() * 2) // usually I/O bound
+
+	g.Go(func() error {
+		return fs.WalkDir(os.DirFS("."), root, func(path string, d fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+			if !d.IsDir() {
+				return nil
+			}
+			if skip[path] {
+				return fs.SkipDir
+			}
+			base := filepath.Base(path)
+			if base == "testdata" || (path != "." && base[0] == '.') {
+				return fs.SkipDir
+			}
+			run(func() error {
+				return update(path)
+			})
+			return nil
+		})
+	})
+	g.Wait()
+	if err := errors.Join(updateErrs...); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func update(dir string) error {
+	readmePath := filepath.Join(dir, "README.md")
+	cur, err := os.ReadFile(readmePath)
+	exists := false
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	if err == nil {
+		exists = true
+		if !isGenerated(cur) {
+			// Do nothing; a human wrote this file.
+			return nil
+		}
+	}
+
+	newContents, err := getNewContent(dir)
+	if err != nil {
+		return err
+	}
+	if newContents == nil {
+		if exists {
+			log.Printf("Deleting %s ...", readmePath)
+			os.Remove(readmePath)
+		}
+		return nil
+	}
+
+	if bytes.Equal(cur, newContents) {
+		return nil
+	}
+	log.Printf("Writing %s ...", readmePath)
+	return os.WriteFile(readmePath, newContents, 0644)
+}
+
+func getNewContent(dir string) (newContent []byte, err error) {
+	dents, err := os.ReadDir(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	generators := []struct {
+		name      string
+		quickTest func(dir string, dents []fs.DirEntry) bool
+		generate  func(dir string) ([]byte, error)
+	}{
+		{"go", hasPkgMainGoFiles, genGoDoc},
+	}
+	for _, gen := range generators {
+		if !gen.quickTest(dir, dents) {
+			continue
+		}
+		newContent, err := gen.generate(dir)
+		if newContent == nil && err == nil {
+			// Generator declined to generate, try next
+			continue
+		}
+		return newContent, err
+	}
+	return nil, nil
+}
+
+func genGoDoc(dir string) ([]byte, error) {
+	abs, err := filepath.Abs(dir)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get absolute path for %q: %w", dir, err)
+	}
+	godoc, err := pkgdoc.PackageDoc(abs)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get package doc for %q: %w", dir, err)
+	}
+	if len(bytes.TrimSpace(godoc)) == 0 {
+		// No godoc; skipping.
+		return nil, nil
+	}
+	if bytes.HasPrefix(godoc, []byte("package ")) {
+		// Not a package main; skipping.
+		return nil, nil
+	}
+	var buf bytes.Buffer
+	io.WriteString(&buf, genHeader)
+	fmt.Fprintf(&buf, "\n# %s\n\n", filepath.Base(dir))
+	buf.Write(godoc)
+
+	if !bytes.Contains(godoc, []byte("## Deploying")) {
+		deployPath := filepath.Join(dir, "deploy.sh")
+		if _, err := os.Stat(deployPath); err == nil {
+			fmt.Fprint(&buf, "\n## Deploying\n\n")
+			if hasBuildkite(dir) {
+				fmt.Fprintf(&buf,
+					"To deploy, run the https://buildkite.com/tailscale/deploy-%s workflow in Buildkite.\n",
+					filepath.Base(dir),
+				)
+			}
+			fmt.Fprintf(&buf, "To deploy manually, run `./%s` from the repo root.\n\n", deployPath)
+		}
+	}
+	return buf.Bytes(), nil
+}
+
+const genHeader = "<!-- README.md auto-generated by misc/genreadme; DO NOT EDIT. (or remove this line) -->\n"
+
+func isGenerated(b []byte) bool { return bytes.HasPrefix(b, []byte(genHeader)) }
+
+func hasBuildkite(dir string) bool {
+	if bkSkip[dir] {
+		return false
+	}
+	_, flyErr := os.Stat(filepath.Join(dir, "fly.toml"))
+	return flyErr != nil
+}
+
+func hasPkgMainGoFiles(dir string, dents []fs.DirEntry) bool {
+	var fset *token.FileSet
+
+	for _, de := range dents {
+		name := de.Name()
+		if !strings.HasSuffix(name, ".go") ||
+			strings.HasSuffix(name, "_test.go") {
+			continue
+		}
+		if fset == nil {
+			fset = token.NewFileSet()
+		}
+
+		path := filepath.Join(dir, name)
+		f, err := os.Open(path)
+		if err != nil {
+			continue
+		}
+		pkgFile, err := parser.ParseFile(fset, "", f, parser.PackageClauseOnly)
+		f.Close()
+		if err != nil {
+			// skip files with parse errors
+			continue
+		}
+
+		return pkgFile.Name.Name == "main"
+	}
+	return false
+}
--- a/tempfork/pkgdoc/pkgdoc.go
+++ b/tempfork/pkgdoc/pkgdoc.go
@ -0,0 +1,198 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package pkgdoc is a library-ified fork of Go's cmd/doc program
+// that only does what we need for misc/genreadme.
+package pkgdoc
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"go/ast"
+	"go/build"
+	"go/doc"
+	"go/parser"
+	"go/token"
+	"io"
+	"io/fs"
+	"log"
+	"slices"
+)
+
+const (
+	punchedCardWidth = 80
+	indent           = "    "
+)
+
+type Package struct {
+	writer   io.Writer    // Destination for output.
+	name     string       // Package name, json for encoding/json.
+	userPath string       // String the user used to find this package.
+	pkg      *ast.Package // Parsed package.
+	file     *ast.File    // Merged from all files in the package
+	doc      *doc.Package
+	build    *build.Package
+	fs       *token.FileSet // Needed for printing.
+	buf      pkgBuffer
+}
+
+func (pkg *Package) ToText(w io.Writer, text, prefix, codePrefix string) {
+	d := pkg.doc.Parser().Parse(text)
+	pr := pkg.doc.Printer()
+	pr.TextPrefix = prefix
+	pr.TextCodePrefix = codePrefix
+	w.Write(pr.Text(d))
+}
+
+// pkgBuffer is a wrapper for bytes.Buffer that prints a package clause the
+// first time Write is called.
+type pkgBuffer struct {
+	pkg     *Package
+	printed bool // Prevent repeated package clauses.
+	bytes.Buffer
+}
+
+func (pb *pkgBuffer) Write(p []byte) (int, error) {
+	pb.packageClause()
+	return pb.Buffer.Write(p)
+}
+
+func (pb *pkgBuffer) packageClause() {
+	if !pb.printed {
+		pb.printed = true
+		// Only show package clause for commands if requested explicitly.
+		if pb.pkg.pkg.Name != "main" {
+			pb.pkg.packageClause()
+		}
+	}
+}
+
+type PackageError string // type returned by pkg.Fatalf.
+
+func (p PackageError) Error() string {
+	return string(p)
+}
+
+// parsePackage turns the build package we found into a parsed package
+// we can then use to generate documentation.
+func parsePackage(writer io.Writer, pkg *build.Package, userPath string) *Package {
+	// include tells parser.ParseDir which files to include.
+	// That means the file must be in the build package's GoFiles or CgoFiles
+	// list only (no tag-ignored files, tests, swig or other non-Go files).
+	include := func(info fs.FileInfo) bool {
+		return slices.Contains(pkg.GoFiles, info.Name()) || slices.Contains(pkg.CgoFiles, info.Name())
+	}
+	fset := token.NewFileSet()
+	pkgs, err := parser.ParseDir(fset, pkg.Dir, include, parser.ParseComments|parser.ImportsOnly)
+	if err != nil {
+		log.Fatal(err)
+	}
+	// Make sure they are all in one package.
+	if len(pkgs) == 0 {
+		log.Fatalf("no source-code package in directory %s", pkg.Dir)
+	}
+	if len(pkgs) > 1 {
+		log.Fatalf("multiple packages in directory %s", pkg.Dir)
+	}
+	astPkg := pkgs[pkg.Name]
+
+	// TODO: go/doc does not include typed constants in the constants
+	// list, which is what we want. For instance, time.Sunday is of type
+	// time.Weekday, so it is defined in the type but not in the
+	// Consts list for the package. This prevents
+	//	go doc time.Sunday
+	// from finding the symbol. Work around this for now, but we
+	// should fix it in go/doc.
+	// A similar story applies to factory functions.
+	mode := doc.AllDecls
+	docPkg := doc.New(astPkg, pkg.ImportPath, mode)
+
+	p := &Package{
+		writer:   writer,
+		name:     pkg.Name,
+		userPath: userPath,
+		pkg:      astPkg,
+		file:     ast.MergePackageFiles(astPkg, 0),
+		doc:      docPkg,
+		build:    pkg,
+		fs:       fset,
+	}
+	p.buf.pkg = p
+	return p
+}
+
+func (pkg *Package) Printf(format string, args ...any) {
+	fmt.Fprintf(&pkg.buf, format, args...)
+}
+
+func (pkg *Package) flush() {
+	_, err := pkg.writer.Write(pkg.buf.Bytes())
+	if err != nil {
+		log.Fatal(err)
+	}
+	pkg.buf.Reset() // Not needed, but it's a flush.
+}
+
+var newlineBytes = []byte("\n\n") // We never ask for more than 2.
+
+// newlines guarantees there are n newlines at the end of the buffer.
+func (pkg *Package) newlines(n int) {
+	for !bytes.HasSuffix(pkg.buf.Bytes(), newlineBytes[:n]) {
+		pkg.buf.WriteRune('\n')
+	}
+}
+
+// packageDoc prints the docs for the package.
+func (pkg *Package) packageDoc() {
+	pkg.Printf("") // Trigger the package clause; we know the package exists.
+	pkg.ToText(&pkg.buf, pkg.doc.Doc, "", indent)
+	pkg.newlines(1)
+
+	pkg.bugs()
+}
+
+// packageClause prints the package clause.
+func (pkg *Package) packageClause() {
+	importPath := pkg.build.ImportComment
+	if importPath == "" {
+		importPath = pkg.build.ImportPath
+	}
+
+	pkg.Printf("package %s // import %q\n\n", pkg.name, importPath)
+}
+
+// bugs prints the BUGS information for the package.
+// TODO: Provide access to TODOs and NOTEs as well (very noisy so off by default)?
+func (pkg *Package) bugs() {
+	if pkg.doc.Notes["BUG"] == nil {
+		return
+	}
+	pkg.Printf("\n")
+	for _, note := range pkg.doc.Notes["BUG"] {
+		pkg.Printf("%s: %v\n", "BUG", note.Body)
+	}
+}
+
+// PackageDoc generates documentation for a package in the given directory.
+func PackageDoc(dir string) ([]byte, error) {
+	var buf bytes.Buffer
+	var writer io.Writer = &buf
+
+	buildPackage, err := build.ImportDir(dir, build.ImportComment)
+	if err != nil {
+		var noGoError *build.NoGoError
+		if errors.As(err, &noGoError) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	userPath := dir
+
+	pkg := parsePackage(writer, buildPackage, userPath)
+	pkg.packageDoc()
+	pkg.flush()
+
+	return buf.Bytes(), nil
+}