mirror of
https://github.com/tailscale/tailscale.git
synced 2026-05-04 11:51:17 +02:00
tool/updateflakes, cmd/nardump: replace update-flake.sh with Go tool
Consolidate go.mod.sri and go.toolchain.rev.sri into a single flakehashes.json file at the repo root, owned by a new Go program at tool/updateflakes. The JSON is consumed by flake.nix via builtins.fromJSON and by any future Go code via the FlakeHashes struct that defines its schema. Each block records its input fingerprint alongside the SRI it produced: the goModSum (a sha256 over go.mod and go.sum) for the vendor block, and the literal rev string from go.toolchain.rev for the toolchain block. updateflakes regenerates a block only when its recorded fingerprint disagrees with the current input. Doing the gating by content rather than file mtimes avoids the usual mtime hazards across git checkouts, clones, and merges. It also means re-runs with no input changes are essentially free, and a re-run that touches only one input pays only for that one block. The two blocks have no shared state -- vendor invokes go mod vendor into one tempdir, toolchain fetches and extracts a tarball into another -- so they run concurrently via errgroup. Cold time is bounded by the slower of the two rather than their sum. Also takes the opportunity to fold the toolchain fetch into a single curl|tar pipeline (no intermediate .tar.gz on disk). Split cmd/nardump into a thin package main and a new package nardump library at cmd/nardump/nardump that holds the NAR encoder and SRI helper. tool/updateflakes imports the library directly rather than building and exec'ing the nardump binary at runtime. The library uses fs.ReadLink (Go 1.25+) instead of os.Readlink, so it no longer requires the caller to chdir into the FS root for symlink targets to resolve. WriteNAR now wraps its writer in a bufio.Writer internally (unless the caller already passed one) and flushes on return, so callers don't pay for tiny writes against slow underlying writers. The cache-busting line in flake.nix and shell.nix is known to live at end of file, so updateCacheBust walks the lines in reverse. make tidy timings on this machine, before: ~14s every run. After: warm (no input changes): 0.05s vendor block stale only: 1.4s toolchain block stale only: 5.0s cold (no flakehashes.json): 5.0s Updates #6845 Change-Id: I0340608798f1614abf147a491bf7c68a198a0db4 Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>
This commit is contained in:
parent
33714211c8
commit
88cb6f58f8
8
.github/workflows/update-flake.yml
vendored
8
.github/workflows/update-flake.yml
vendored
@ -23,8 +23,8 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Run update-flakes
|
||||
run: ./update-flake.sh
|
||||
- name: Run updateflakes
|
||||
run: ./tool/go run ./tool/updateflakes
|
||||
|
||||
- name: Get access token
|
||||
uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0
|
||||
@ -41,8 +41,8 @@ jobs:
|
||||
author: Flakes Updater <noreply+flakes-updater@tailscale.com>
|
||||
committer: Flakes Updater <noreply+flakes-updater@tailscale.com>
|
||||
branch: flakes
|
||||
commit-message: "go.mod.sri: update SRI hash for go.mod changes"
|
||||
title: "go.mod.sri: update SRI hash for go.mod changes"
|
||||
commit-message: "flakehashes.json: update SRI hash for go.mod changes"
|
||||
title: "flakehashes.json: update SRI hash for go.mod changes"
|
||||
body: Triggered by ${{ github.repository }}@${{ github.sha }}
|
||||
signoff: true
|
||||
delete-branch: true
|
||||
|
||||
2
Makefile
2
Makefile
@ -10,7 +10,7 @@ vet: ## Run go vet
|
||||
|
||||
tidy: ## Run go mod tidy and update nix flake hashes
|
||||
./tool/go mod tidy
|
||||
./update-flake.sh
|
||||
./tool/go run ./tool/updateflakes
|
||||
|
||||
lint: ## Run golangci-lint
|
||||
./tool/go run github.com/golangci/golangci-lint/cmd/golangci-lint run
|
||||
|
||||
@ -9,22 +9,13 @@
|
||||
// git-pull-oss.sh having Nix available.
|
||||
package main
|
||||
|
||||
// For the format, see:
|
||||
// See https://gist.github.com/jbeda/5c79d2b1434f0018d693
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"log"
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
|
||||
"tailscale.com/cmd/nardump/nardump"
|
||||
)
|
||||
|
||||
var sri = flag.Bool("sri", false, "print SRI")
|
||||
@ -34,167 +25,16 @@ func main() {
|
||||
if flag.NArg() != 1 {
|
||||
log.Fatal("usage: nardump <dir>")
|
||||
}
|
||||
arg := flag.Arg(0)
|
||||
if err := os.Chdir(arg); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
fsys := os.DirFS(flag.Arg(0))
|
||||
if *sri {
|
||||
hash := sha256.New()
|
||||
if err := writeNAR(hash, os.DirFS(".")); err != nil {
|
||||
s, err := nardump.SRI(fsys)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
fmt.Printf("sha256-%s\n", base64.StdEncoding.EncodeToString(hash.Sum(nil)))
|
||||
fmt.Println(s)
|
||||
return
|
||||
}
|
||||
bw := bufio.NewWriter(os.Stdout)
|
||||
if err := writeNAR(bw, os.DirFS(".")); err != nil {
|
||||
if err := nardump.WriteNAR(os.Stdout, fsys); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
bw.Flush()
|
||||
}
|
||||
|
||||
// writeNARError is a sentinel panic type that's recovered by writeNAR
|
||||
// and converted into the wrapped error.
|
||||
type writeNARError struct{ err error }
|
||||
|
||||
// narWriter writes NAR files.
|
||||
type narWriter struct {
|
||||
w io.Writer
|
||||
fs fs.FS
|
||||
}
|
||||
|
||||
// writeNAR writes a NAR file to w from the root of fs.
|
||||
func writeNAR(w io.Writer, fs fs.FS) (err error) {
|
||||
defer func() {
|
||||
if e := recover(); e != nil {
|
||||
if we, ok := e.(writeNARError); ok {
|
||||
err = we.err
|
||||
return
|
||||
}
|
||||
panic(e)
|
||||
}
|
||||
}()
|
||||
nw := &narWriter{w: w, fs: fs}
|
||||
nw.str("nix-archive-1")
|
||||
return nw.writeDir(".")
|
||||
}
|
||||
|
||||
func (nw *narWriter) writeDir(dirPath string) error {
|
||||
ents, err := fs.ReadDir(nw.fs, dirPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sort.Slice(ents, func(i, j int) bool {
|
||||
return ents[i].Name() < ents[j].Name()
|
||||
})
|
||||
nw.str("(")
|
||||
nw.str("type")
|
||||
nw.str("directory")
|
||||
for _, ent := range ents {
|
||||
nw.str("entry")
|
||||
nw.str("(")
|
||||
nw.str("name")
|
||||
nw.str(ent.Name())
|
||||
nw.str("node")
|
||||
mode := ent.Type()
|
||||
sub := path.Join(dirPath, ent.Name())
|
||||
var err error
|
||||
switch {
|
||||
case mode.IsDir():
|
||||
err = nw.writeDir(sub)
|
||||
case mode.IsRegular():
|
||||
err = nw.writeRegular(sub)
|
||||
case mode&os.ModeSymlink != 0:
|
||||
err = nw.writeSymlink(sub)
|
||||
default:
|
||||
return fmt.Errorf("unsupported file type %v at %q", sub, mode)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
nw.str(")")
|
||||
}
|
||||
nw.str(")")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (nw *narWriter) writeRegular(path string) error {
|
||||
nw.str("(")
|
||||
nw.str("type")
|
||||
nw.str("regular")
|
||||
fi, err := fs.Stat(nw.fs, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if fi.Mode()&0111 != 0 {
|
||||
nw.str("executable")
|
||||
nw.str("")
|
||||
}
|
||||
contents, err := fs.ReadFile(nw.fs, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
nw.str("contents")
|
||||
if err := writeBytes(nw.w, contents); err != nil {
|
||||
return err
|
||||
}
|
||||
nw.str(")")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (nw *narWriter) writeSymlink(path string) error {
|
||||
nw.str("(")
|
||||
nw.str("type")
|
||||
nw.str("symlink")
|
||||
nw.str("target")
|
||||
// broken symlinks are valid in a nar
|
||||
// given we do os.chdir(dir) and os.dirfs(".") above
|
||||
// readlink now resolves relative links even if they are broken
|
||||
link, err := os.Readlink(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
nw.str(link)
|
||||
nw.str(")")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (nw *narWriter) str(s string) {
|
||||
if err := writeString(nw.w, s); err != nil {
|
||||
panic(writeNARError{err})
|
||||
}
|
||||
}
|
||||
|
||||
func writeString(w io.Writer, s string) error {
|
||||
var buf [8]byte
|
||||
binary.LittleEndian.PutUint64(buf[:], uint64(len(s)))
|
||||
if _, err := w.Write(buf[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.WriteString(w, s); err != nil {
|
||||
return err
|
||||
}
|
||||
return writePad(w, len(s))
|
||||
}
|
||||
|
||||
func writeBytes(w io.Writer, b []byte) error {
|
||||
var buf [8]byte
|
||||
binary.LittleEndian.PutUint64(buf[:], uint64(len(b)))
|
||||
if _, err := w.Write(buf[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := w.Write(b); err != nil {
|
||||
return err
|
||||
}
|
||||
return writePad(w, len(b))
|
||||
}
|
||||
|
||||
func writePad(w io.Writer, n int) error {
|
||||
pad := n % 8
|
||||
if pad == 0 {
|
||||
return nil
|
||||
}
|
||||
var zeroes [8]byte
|
||||
_, err := w.Write(zeroes[:8-pad])
|
||||
return err
|
||||
}
|
||||
|
||||
193
cmd/nardump/nardump/nardump.go
Normal file
193
cmd/nardump/nardump/nardump.go
Normal file
@ -0,0 +1,193 @@
|
||||
// Copyright (c) Tailscale Inc & contributors
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
// Package nardump writes a NAR (Nix Archive) representation of an
|
||||
// fs.FS to an io.Writer, or summarizes it as a Subresource Integrity
|
||||
// hash, as used by Nix flake.nix vendor and toolchain hashes.
|
||||
//
|
||||
// For the format, see:
|
||||
// https://gist.github.com/jbeda/5c79d2b1434f0018d693
|
||||
package nardump
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"path"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// WriteNAR writes a NAR-encoded representation of fsys, rooted at
|
||||
// the FS root, to w.
|
||||
//
|
||||
// The encoder issues many small writes; if w is not already a
|
||||
// *bufio.Writer, WriteNAR wraps it in one and flushes on return so
|
||||
// the caller doesn't have to.
|
||||
//
|
||||
// fsys must implement fs.ReadLinkFS to encode any symlinks it
|
||||
// contains; os.DirFS satisfies this on Go 1.25+.
|
||||
func WriteNAR(w io.Writer, fsys fs.FS) (err error) {
|
||||
defer func() {
|
||||
if e := recover(); e != nil {
|
||||
if we, ok := e.(writeNARError); ok {
|
||||
err = we.err
|
||||
return
|
||||
}
|
||||
panic(e)
|
||||
}
|
||||
}()
|
||||
bw, ok := w.(*bufio.Writer)
|
||||
if !ok {
|
||||
bw = bufio.NewWriter(w)
|
||||
defer func() {
|
||||
if flushErr := bw.Flush(); err == nil {
|
||||
err = flushErr
|
||||
}
|
||||
}()
|
||||
}
|
||||
nw := &narWriter{w: bw, fs: fsys}
|
||||
nw.str("nix-archive-1")
|
||||
return nw.writeDir(".")
|
||||
}
|
||||
|
||||
// SRI returns the Subresource Integrity hash of the NAR encoding of
|
||||
// fsys, in the form "sha256-<base64>". This is the format Nix
|
||||
// expects for vendorHash and similar fields.
|
||||
func SRI(fsys fs.FS) (string, error) {
|
||||
h := sha256.New()
|
||||
if err := WriteNAR(h, fsys); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "sha256-" + base64.StdEncoding.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
// writeNARError is a sentinel panic type that's recovered by
|
||||
// WriteNAR and converted into the wrapped error.
|
||||
type writeNARError struct{ err error }
|
||||
|
||||
// narWriter writes NAR files.
|
||||
type narWriter struct {
|
||||
w io.Writer
|
||||
fs fs.FS
|
||||
}
|
||||
|
||||
func (nw *narWriter) writeDir(dirPath string) error {
|
||||
ents, err := fs.ReadDir(nw.fs, dirPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sort.Slice(ents, func(i, j int) bool {
|
||||
return ents[i].Name() < ents[j].Name()
|
||||
})
|
||||
nw.str("(")
|
||||
nw.str("type")
|
||||
nw.str("directory")
|
||||
for _, ent := range ents {
|
||||
nw.str("entry")
|
||||
nw.str("(")
|
||||
nw.str("name")
|
||||
nw.str(ent.Name())
|
||||
nw.str("node")
|
||||
mode := ent.Type()
|
||||
sub := path.Join(dirPath, ent.Name())
|
||||
var err error
|
||||
switch {
|
||||
case mode.IsDir():
|
||||
err = nw.writeDir(sub)
|
||||
case mode.IsRegular():
|
||||
err = nw.writeRegular(sub)
|
||||
case mode&fs.ModeSymlink != 0:
|
||||
err = nw.writeSymlink(sub)
|
||||
default:
|
||||
return fmt.Errorf("unsupported file type %v at %q", sub, mode)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
nw.str(")")
|
||||
}
|
||||
nw.str(")")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (nw *narWriter) writeRegular(p string) error {
|
||||
nw.str("(")
|
||||
nw.str("type")
|
||||
nw.str("regular")
|
||||
fi, err := fs.Stat(nw.fs, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if fi.Mode()&0111 != 0 {
|
||||
nw.str("executable")
|
||||
nw.str("")
|
||||
}
|
||||
contents, err := fs.ReadFile(nw.fs, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
nw.str("contents")
|
||||
if err := writeBytes(nw.w, contents); err != nil {
|
||||
return err
|
||||
}
|
||||
nw.str(")")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (nw *narWriter) writeSymlink(p string) error {
|
||||
nw.str("(")
|
||||
nw.str("type")
|
||||
nw.str("symlink")
|
||||
nw.str("target")
|
||||
link, err := fs.ReadLink(nw.fs, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
nw.str(link)
|
||||
nw.str(")")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (nw *narWriter) str(s string) {
|
||||
if err := writeString(nw.w, s); err != nil {
|
||||
panic(writeNARError{err})
|
||||
}
|
||||
}
|
||||
|
||||
func writeString(w io.Writer, s string) error {
|
||||
var buf [8]byte
|
||||
binary.LittleEndian.PutUint64(buf[:], uint64(len(s)))
|
||||
if _, err := w.Write(buf[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.WriteString(w, s); err != nil {
|
||||
return err
|
||||
}
|
||||
return writePad(w, len(s))
|
||||
}
|
||||
|
||||
func writeBytes(w io.Writer, b []byte) error {
|
||||
var buf [8]byte
|
||||
binary.LittleEndian.PutUint64(buf[:], uint64(len(b)))
|
||||
if _, err := w.Write(buf[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := w.Write(b); err != nil {
|
||||
return err
|
||||
}
|
||||
return writePad(w, len(b))
|
||||
}
|
||||
|
||||
func writePad(w io.Writer, n int) error {
|
||||
pad := n % 8
|
||||
if pad == 0 {
|
||||
return nil
|
||||
}
|
||||
var zeroes [8]byte
|
||||
_, err := w.Write(zeroes[:8-pad])
|
||||
return err
|
||||
}
|
||||
55
cmd/nardump/nardump/nardump_test.go
Normal file
55
cmd/nardump/nardump/nardump_test.go
Normal file
@ -0,0 +1,55 @@
|
||||
// Copyright (c) Tailscale Inc & contributors
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
package nardump
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// setupTmpdir sets up a known golden layout, covering all allowed file/folder types in a nar.
|
||||
func setupTmpdir(t *testing.T) string {
|
||||
t.Helper()
|
||||
tmpdir := t.TempDir()
|
||||
must := func(err error) {
|
||||
t.Helper()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
must(os.MkdirAll(filepath.Join(tmpdir, "sub/dir"), 0755))
|
||||
must(os.Symlink("brokenfile", filepath.Join(tmpdir, "brokenlink")))
|
||||
must(os.Symlink("sub/dir", filepath.Join(tmpdir, "dirl")))
|
||||
must(os.Symlink("/abs/nonexistentdir", filepath.Join(tmpdir, "dirb")))
|
||||
f, err := os.Create(filepath.Join(tmpdir, "sub/dir/file1"))
|
||||
must(err)
|
||||
f.Close()
|
||||
f, err = os.Create(filepath.Join(tmpdir, "file2m"))
|
||||
must(err)
|
||||
must(f.Truncate(2 * 1024 * 1024))
|
||||
f.Close()
|
||||
must(os.Symlink("../file2m", filepath.Join(tmpdir, "sub/goodlink")))
|
||||
return tmpdir
|
||||
}
|
||||
|
||||
func TestWriteNAR(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
// Skip test on Windows as the Nix package manager is not supported on this platform
|
||||
t.Skip("nix package manager is not available on Windows")
|
||||
}
|
||||
dir := setupTmpdir(t)
|
||||
// obtained via `nix-store --dump /tmp/... | sha256sum` of the above test dir
|
||||
const expected = "727613a36f41030e93a4abf2649c3ec64a2757ccff364e3f6f7d544eb976e442"
|
||||
h := sha256.New()
|
||||
if err := WriteNAR(h, os.DirFS(dir)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := fmt.Sprintf("%x", h.Sum(nil)); got != expected {
|
||||
t.Fatalf("sha256sum of nar: got %s, want %s", got, expected)
|
||||
}
|
||||
}
|
||||
@ -1,52 +0,0 @@
|
||||
// Copyright (c) Tailscale Inc & contributors
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"os"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// setupTmpdir sets up a known golden layout, covering all allowed file/folder types in a nar
|
||||
func setupTmpdir(t *testing.T) string {
|
||||
tmpdir := t.TempDir()
|
||||
pwd, _ := os.Getwd()
|
||||
os.Chdir(tmpdir)
|
||||
defer os.Chdir(pwd)
|
||||
os.MkdirAll("sub/dir", 0755)
|
||||
os.Symlink("brokenfile", "brokenlink")
|
||||
os.Symlink("sub/dir", "dirl")
|
||||
os.Symlink("/abs/nonexistentdir", "dirb")
|
||||
os.Create("sub/dir/file1")
|
||||
f, _ := os.Create("file2m")
|
||||
_ = f.Truncate(2 * 1024 * 1024)
|
||||
f.Close()
|
||||
os.Symlink("../file2m", "sub/goodlink")
|
||||
return tmpdir
|
||||
}
|
||||
|
||||
func TestWriteNar(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
// Skip test on Windows as the Nix package manager is not supported on this platform
|
||||
t.Skip("nix package manager is not available on Windows")
|
||||
}
|
||||
dir := setupTmpdir(t)
|
||||
t.Run("nar", func(t *testing.T) {
|
||||
// obtained via `nix-store --dump /tmp/... | sha256sum` of the above test dir
|
||||
expected := "727613a36f41030e93a4abf2649c3ec64a2757ccff364e3f6f7d544eb976e442"
|
||||
h := sha256.New()
|
||||
os.Chdir(dir)
|
||||
err := writeNAR(h, os.DirFS("."))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hash := fmt.Sprintf("%x", h.Sum(nil))
|
||||
if expected != hash {
|
||||
t.Fatal("sha256sum of nar not matched", hash, expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
@ -48,7 +48,8 @@
|
||||
}: let
|
||||
goVersion = nixpkgs.lib.fileContents ./go.toolchain.version;
|
||||
toolChainRev = nixpkgs.lib.fileContents ./go.toolchain.rev;
|
||||
gitHash = nixpkgs.lib.fileContents ./go.toolchain.rev.sri;
|
||||
flakeHashes = builtins.fromJSON (builtins.readFile ./flakehashes.json);
|
||||
gitHash = flakeHashes.toolchain.sri;
|
||||
eachSystem = f:
|
||||
nixpkgs.lib.genAttrs (import systems) (system:
|
||||
f (import nixpkgs {
|
||||
@ -103,7 +104,7 @@
|
||||
name = "tailscale";
|
||||
pname = "tailscale";
|
||||
src = ./.;
|
||||
vendorHash = pkgs.lib.fileContents ./go.mod.sri;
|
||||
vendorHash = flakeHashes.vendor.sri;
|
||||
nativeBuildInputs = [pkgs.makeWrapper pkgs.installShellFiles];
|
||||
ldflags = ["-X tailscale.com/version.gitCommitStamp=${tailscaleRev}"];
|
||||
env.CGO_ENABLED = 0;
|
||||
|
||||
10
flakehashes.json
Normal file
10
flakehashes.json
Normal file
@ -0,0 +1,10 @@
|
||||
{
|
||||
"toolchain": {
|
||||
"rev": "dfe2a5fd8ee2e68b08ce5ff259269f50ecadf2f4",
|
||||
"sri": "sha256-pCvFNTFuvhSBb5O+PPuilaowP4tXcCOP1NgYUDJTcJU="
|
||||
},
|
||||
"vendor": {
|
||||
"goModSum": "sha256-lgxpp/5OxeUwveDyHvBks97hQmLMKkbWd0bZ9ktbhFE=",
|
||||
"sri": "sha256-ruRbOB2W9snyOYY0+6OD5IndI/JJKqrhTuPlBsKikRc="
|
||||
}
|
||||
}
|
||||
@ -1 +0,0 @@
|
||||
sha256-ruRbOB2W9snyOYY0+6OD5IndI/JJKqrhTuPlBsKikRc=
|
||||
@ -1 +0,0 @@
|
||||
sha256-pCvFNTFuvhSBb5O+PPuilaowP4tXcCOP1NgYUDJTcJU=
|
||||
@ -46,15 +46,15 @@ if [ "${TS_GO_NEXT:-}" != "1" ]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# Only update go.toolchain.version and go.toolchain.rev.sri for the main toolchain,
|
||||
# Only update go.toolchain.version and flakehashes.json for the main toolchain,
|
||||
# skipping it if TS_GO_NEXT=1. Those two files are only used by Nix, and as of 2026-01-26
|
||||
# don't yet support TS_GO_NEXT=1 with flake.nix or in our corp CI.
|
||||
if [ "${TS_GO_NEXT:-}" != "1" ]; then
|
||||
./tool/go version 2>/dev/null | awk '{print $3}' | sed 's/^go//' > go.toolchain.version
|
||||
./tool/go mod edit -go "$(cat go.toolchain.version)"
|
||||
./update-flake.sh
|
||||
./tool/go run ./tool/updateflakes
|
||||
fi
|
||||
|
||||
if [ -n "$(git diff-index --name-only HEAD -- "$go_toolchain_rev_file" go.toolchain.next.rev go.toolchain.rev.sri go.toolchain.version)" ]; then
|
||||
if [ -n "$(git diff-index --name-only HEAD -- "$go_toolchain_rev_file" go.toolchain.next.rev flakehashes.json go.toolchain.version)" ]; then
|
||||
echo "pull-toolchain.sh: changes imported. Use git commit to make them permanent." >&2
|
||||
fi
|
||||
|
||||
264
tool/updateflakes/updateflakes.go
Normal file
264
tool/updateflakes/updateflakes.go
Normal file
@ -0,0 +1,264 @@
|
||||
// Copyright (c) Tailscale Inc & contributors
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
// updateflakes regenerates flakehashes.json, the file that records
|
||||
// the Nix SRI hashes for the Go module vendor tree and the Tailscale
|
||||
// Go toolchain tarball.
|
||||
//
|
||||
// The file is content-addressed: each block records the input
|
||||
// fingerprint that produced its SRI, and updateflakes only
|
||||
// regenerates a block when the current input differs from the
|
||||
// recorded fingerprint. As a result, repeat runs with no input
|
||||
// changes are no-ops.
|
||||
//
|
||||
// Run from the repo root:
|
||||
//
|
||||
// ./tool/go run ./tool/updateflakes
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/sync/errgroup"
|
||||
"tailscale.com/cmd/nardump/nardump"
|
||||
)
|
||||
|
||||
const (
|
||||
hashesFile = "flakehashes.json"
|
||||
goModFile = "go.mod"
|
||||
goSumFile = "go.sum"
|
||||
toolchainRevFile = "go.toolchain.rev"
|
||||
flakeNixFile = "flake.nix"
|
||||
shellNixFile = "shell.nix"
|
||||
cacheBustPrefix = "# nix-direnv cache busting line:"
|
||||
)
|
||||
|
||||
// FlakeHashes is the on-disk schema of flakehashes.json. It is also
|
||||
// consumed directly by flake.nix via builtins.fromJSON, so changes
|
||||
// to the JSON shape must be coordinated with flake.nix.
|
||||
type FlakeHashes struct {
|
||||
Toolchain ToolchainHash `json:"toolchain"`
|
||||
Vendor VendorHash `json:"vendor"`
|
||||
}
|
||||
|
||||
// ToolchainHash records the SRI of the Tailscale Go toolchain
|
||||
// tarball. Rev is the value in go.toolchain.rev that produced SRI.
|
||||
type ToolchainHash struct {
|
||||
Rev string `json:"rev"`
|
||||
SRI string `json:"sri"`
|
||||
}
|
||||
|
||||
// VendorHash records the SRI of `go mod vendor` output. GoModSum is a
|
||||
// fingerprint of go.mod and go.sum that produced SRI.
|
||||
type VendorHash struct {
|
||||
GoModSum string `json:"goModSum"`
|
||||
SRI string `json:"sri"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
if err := run(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func run() error {
|
||||
have, err := loadHashes()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
want := have
|
||||
|
||||
rev, err := readTrim(toolchainRevFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
wantToolchain := have.Toolchain.Rev != rev || have.Toolchain.SRI == ""
|
||||
|
||||
goModSum, err := goModFingerprint()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
wantVendor := have.Vendor.GoModSum != goModSum || have.Vendor.SRI == ""
|
||||
|
||||
var (
|
||||
newToolchain ToolchainHash
|
||||
newVendor VendorHash
|
||||
)
|
||||
var g errgroup.Group
|
||||
if wantToolchain {
|
||||
g.Go(func() error {
|
||||
sri, err := hashToolchain(rev)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newToolchain = ToolchainHash{Rev: rev, SRI: sri}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if wantVendor {
|
||||
g.Go(func() error {
|
||||
sri, err := hashVendor()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newVendor = VendorHash{GoModSum: goModSum, SRI: sri}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if err := g.Wait(); err != nil {
|
||||
return err
|
||||
}
|
||||
if wantToolchain {
|
||||
want.Toolchain = newToolchain
|
||||
}
|
||||
if wantVendor {
|
||||
want.Vendor = newVendor
|
||||
}
|
||||
|
||||
if want != have {
|
||||
if err := writeHashes(want); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// nix-direnv only watches the top-level nix files for changes,
|
||||
// so when a referenced hash changes we must also tickle
|
||||
// flake.nix and shell.nix to force re-evaluation.
|
||||
for _, f := range []string{flakeNixFile, shellNixFile} {
|
||||
if err := updateCacheBust(f, want.Vendor.SRI); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadHashes() (FlakeHashes, error) {
|
||||
var h FlakeHashes
|
||||
data, err := os.ReadFile(hashesFile)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return h, nil
|
||||
}
|
||||
if err != nil {
|
||||
return h, err
|
||||
}
|
||||
if err := json.Unmarshal(data, &h); err != nil {
|
||||
return h, fmt.Errorf("parse %s: %w", hashesFile, err)
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
|
||||
func writeHashes(h FlakeHashes) error {
|
||||
b, err := json.MarshalIndent(h, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b = append(b, '\n')
|
||||
return os.WriteFile(hashesFile, b, 0644)
|
||||
}
|
||||
|
||||
func readTrim(path string) (string, error) {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strings.TrimSpace(string(b)), nil
|
||||
}
|
||||
|
||||
// goModFingerprint returns a content fingerprint of go.mod and go.sum
|
||||
// that changes whenever either file changes.
|
||||
func goModFingerprint() (string, error) {
|
||||
h := sha256.New()
|
||||
for _, f := range []string{goModFile, goSumFile} {
|
||||
b, err := os.ReadFile(f)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
fmt.Fprintf(h, "%s %d\n", f, len(b))
|
||||
h.Write(b)
|
||||
}
|
||||
return "sha256-" + base64.StdEncoding.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
func hashVendor() (string, error) {
|
||||
out, err := os.MkdirTemp("", "nar-vendor-")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// `go mod vendor -o` requires the destination to not already exist.
|
||||
if err := os.Remove(out); err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer os.RemoveAll(out)
|
||||
|
||||
cmd := exec.Command("./tool/go", "mod", "vendor", "-o", out)
|
||||
cmd.Env = append(os.Environ(), "GOWORK=off")
|
||||
cmd.Stderr = os.Stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
return "", fmt.Errorf("go mod vendor: %w", err)
|
||||
}
|
||||
return nardump.SRI(os.DirFS(out))
|
||||
}
|
||||
|
||||
func hashToolchain(rev string) (string, error) {
|
||||
out, err := os.MkdirTemp("", "nar-toolchain-")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer os.RemoveAll(out)
|
||||
|
||||
url := fmt.Sprintf("https://github.com/tailscale/go/archive/%s.tar.gz", rev)
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("fetching %s: %w", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("fetching %s: %s", url, resp.Status)
|
||||
}
|
||||
|
||||
tar := exec.Command("tar", "-xz", "-C", out)
|
||||
tar.Stdin = resp.Body
|
||||
tar.Stderr = os.Stderr
|
||||
if err := tar.Run(); err != nil {
|
||||
return "", fmt.Errorf("extracting toolchain tarball: %w", err)
|
||||
}
|
||||
return nardump.SRI(os.DirFS(filepath.Join(out, "go-"+rev)))
|
||||
}
|
||||
|
||||
// updateCacheBust rewrites the "# nix-direnv cache busting line"
|
||||
// in path to embed sri so nix-direnv re-evaluates when the SRI
|
||||
// changes. The line lives at end of file, so walk in reverse.
|
||||
func updateCacheBust(path, sri string) error {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
want := []byte(cacheBustPrefix + " " + sri)
|
||||
lines := bytes.Split(b, []byte("\n"))
|
||||
for i := len(lines) - 1; i >= 0; i-- {
|
||||
line := lines[i]
|
||||
if !bytes.HasPrefix(line, []byte(cacheBustPrefix)) {
|
||||
continue
|
||||
}
|
||||
if bytes.Equal(line, want) {
|
||||
return nil
|
||||
}
|
||||
lines[i] = want
|
||||
return os.WriteFile(path, bytes.Join(lines, []byte("\n")), 0644)
|
||||
}
|
||||
return fmt.Errorf("%s: missing %q line", path, cacheBustPrefix)
|
||||
}
|
||||
@ -1,26 +0,0 @@
|
||||
#!/bin/sh
|
||||
# Updates SRI hashes for flake.nix.
|
||||
|
||||
set -eu
|
||||
|
||||
OUT=$(mktemp -d -t nar-hash-XXXXXX)
|
||||
rm -rf "$OUT"
|
||||
|
||||
./tool/go mod vendor -o "$OUT"
|
||||
./tool/go run tailscale.com/cmd/nardump --sri "$OUT" >go.mod.sri
|
||||
rm -rf "$OUT"
|
||||
|
||||
GOOUT=$(mktemp -d -t gocross-XXXXXX)
|
||||
GOREV=$(xargs < ./go.toolchain.rev)
|
||||
TARBALL="$GOOUT/go-$GOREV.tar.gz"
|
||||
curl -Ls -o "$TARBALL" "https://github.com/tailscale/go/archive/$GOREV.tar.gz"
|
||||
tar -xzf "$TARBALL" -C "$GOOUT"
|
||||
./tool/go run tailscale.com/cmd/nardump --sri "$GOOUT/go-$GOREV" > go.toolchain.rev.sri
|
||||
rm -rf "$GOOUT"
|
||||
|
||||
# nix-direnv only watches the top-level nix file for changes. As a
|
||||
# result, when we change a referenced SRI file, we have to cause some
|
||||
# change to shell.nix and flake.nix as well, so that nix-direnv
|
||||
# notices and reevaluates everything. Sigh.
|
||||
perl -pi -e "s,# nix-direnv cache busting line:.*,# nix-direnv cache busting line: $(cat go.mod.sri)," shell.nix
|
||||
perl -pi -e "s,# nix-direnv cache busting line:.*,# nix-direnv cache busting line: $(cat go.mod.sri)," flake.nix
|
||||
Loading…
x
Reference in New Issue
Block a user