[VAULT-32028] pipeline(github): add sync branches sub-command (#31252)

Add a new `pipeline github sync branches` command that can synchronize
two branches. We'll use this to synchronize the
`hashicorp/vault-enterprise/ce/*` branches with `hashicorp/vault/*`.

As the community repository is effectively a mirror of what is hosted in
Enterprise, a scheduled sync cadence is probably fine. Eventually we'll
hook the workflow and sync into the release pipeline to ensure that
`hashicorp/vault` branches are up-to-date when cutting community
releases.

As part of this I also fixed a few static analysis issues that popped up
when running `golangci-lint` and fixed a few smaller bugs.

Signed-off-by: Ryan Cragun <me@ryan.ec>

tools/pipeline/internal/cmd/github.go

@@ -40,6 +40,7 @@ func newGithubCmd() *cobra.Command {
 	githubCmd.AddCommand(newGithubCopyCmd())
 	githubCmd.AddCommand(newGithubCreateCmd())
 	githubCmd.AddCommand(newGithubListCmd())
+	githubCmd.AddCommand(newGithubSyncCmd())
 
 	return githubCmd
 }

tools/pipeline/internal/cmd/github_copy_pull_request.go

@@ -68,7 +68,7 @@ func runCopyGithubPullRequestCmd(cmd *cobra.Command, args []string) error {
 		}
 		fmt.Println(string(b))
 	default:
-		fmt.Println(res.ToTable(err))
+		fmt.Println(res.ToTable(err).Render())
 	}
 
 	return err

tools/pipeline/internal/cmd/github_sync.go

@@ -0,0 +1,19 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+)
+
+func newGithubSyncCmd() *cobra.Command {
+	syncCmd := &cobra.Command{
+		Use:   "sync",
+		Short: "Github sync commands",
+		Long:  "Github sync commands",
+	}
+	syncCmd.AddCommand(newSyncGithubBranchCmd())
+
+	return syncCmd
+}

tools/pipeline/internal/cmd/github_sync_branch.go

@@ -0,0 +1,64 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/hashicorp/vault/tools/pipeline/internal/pkg/github"
+	"github.com/spf13/cobra"
+)
+
+var syncGithubBranchReq = github.SyncBranchReq{}
+
+func newSyncGithubBranchCmd() *cobra.Command {
+	syncBranchCmd := &cobra.Command{
+		Use:   "branch",
+		Short: "Sync a branch between two repositories",
+		Long:  "Sync a branch between two repositories by merging the --from branch into the --to branch and pushing the result on success",
+		RunE:  runSyncGithubBranchCmd,
+	}
+
+	syncBranchCmd.PersistentFlags().StringVar(&syncGithubBranchReq.FromOrigin, "from-origin", "from", "The origin name to use for the from-branch")
+	syncBranchCmd.PersistentFlags().StringVar(&syncGithubBranchReq.FromOwner, "from-owner", "hashicorp", "The Github organization hosting the from branch")
+	syncBranchCmd.PersistentFlags().StringVar(&syncGithubBranchReq.FromRepo, "from-repo", "vault-enterprise", "The Github repository to sync from")
+	syncBranchCmd.PersistentFlags().StringVar(&syncGithubBranchReq.FromBranch, "from-branch", "", "The name of the branch we want to sync from")
+	syncBranchCmd.PersistentFlags().StringVar(&syncGithubBranchReq.ToOrigin, "to-origin", "to", "The origin name to use for the to-branch")
+	syncBranchCmd.PersistentFlags().StringVar(&syncGithubBranchReq.ToOwner, "to-owner", "hashicorp", "The Github organization hosting the to branch")
+	syncBranchCmd.PersistentFlags().StringVar(&syncGithubBranchReq.ToRepo, "to-repo", "vault-enterprise", "The Github repository to sync to")
+	syncBranchCmd.PersistentFlags().StringVar(&syncGithubBranchReq.ToBranch, "to-branch", "", "The name of the branch we want to sync to")
+	syncBranchCmd.PersistentFlags().StringVarP(&syncGithubBranchReq.RepoDir, "repo-dir", "d", "", "The path to the vault repository dir. If not set a temporary directory will be used")
+
+	err := syncBranchCmd.MarkPersistentFlagRequired("from-branch")
+	if err != nil {
+		panic(err)
+	}
+	err = syncBranchCmd.MarkPersistentFlagRequired("to-branch")
+	if err != nil {
+		panic(err)
+	}
+
+	return syncBranchCmd
+}
+
+func runSyncGithubBranchCmd(cmd *cobra.Command, args []string) error {
+	cmd.SilenceUsage = true // Don't spam the usage on failure
+
+	res, err := syncGithubBranchReq.Run(context.TODO(), githubCmdState.Github, githubCmdState.Git)
+
+	switch rootCfg.format {
+	case "json":
+		b, err1 := res.ToJSON()
+		if err1 != nil {
+			return errors.Join(err, err1)
+		}
+		fmt.Println(string(b))
+	default:
+		fmt.Println(res.ToTable(err).Render())
+	}
+
+	return err
+}

tools/pipeline/internal/pkg/git/client.go

@@ -12,7 +12,6 @@ import (
 	"net/url"
 	"os"
 	"os/exec"
-	oexec "os/exec"
 	"strings"
 	"sync"
 
@@ -99,7 +98,7 @@ func (c *Client) Exec(ctx context.Context, subCmd string, opts OptStringer) (*Ex
 		env = append(env, res.Env...)
 	}
 
-	cmd := oexec.Command("git", append([]string{subCmd}, opts.Strings()...)...)
+	cmd := exec.Command("git", append([]string{subCmd}, opts.Strings()...)...)
 	cmd.Env = env
 	res.Cmd = cmd.String()
 	ctx = slogctx.Append(ctx, slog.String("cmd", cmd.String()))

tools/pipeline/internal/pkg/github/copy_pull_request.go

@@ -78,17 +78,9 @@ func (r *CopyPullRequestReq) Run(
 			res = &CopyPullRequestRes{Request: r}
 		}
 
-		// Figure out the comment body. Worst case it ought to be whatever error
-		// we've returned.
-		var body string
-		if err != nil {
-			body = err.Error()
-		}
-
 		// Set any known errors on the response before we create a comment, as the
 		// error will be used in the comment body if present.
 		err = errors.Join(err, os.Chdir(initialDir))
-		body = res.CommentBody(err)
 		var err1 error
 		res.Comment, err1 = createPullRequestComment(
 			ctx,
@@ -96,7 +88,7 @@ func (r *CopyPullRequestReq) Run(
 			r.FromOwner,
 			r.FromRepo,
 			int(r.PullNumber),
-			body,
+			res.CommentBody(err),
 		)
 
 		// Set our finalized error on our response and also update our returned error
@@ -118,7 +110,7 @@ func (r *CopyPullRequestReq) Run(
 		return res, err
 	}
 	if tmpDir {
-		// defer os.RemoveAll(r.RepoDir)
+		defer os.RemoveAll(r.RepoDir)
 	}
 
 	// Get our pull request details
@@ -332,7 +324,7 @@ func (r *CopyPullRequestReq) Validate(ctx context.Context) error {
 	}
 
 	if r.FromRepo == "" {
-		return errors.New("no github repository has been provided")
+		return errors.New("no github from repository has been provided")
 	}
 
 	if r.ToOrigin == "" {

tools/pipeline/internal/pkg/github/create_backport.go

@@ -15,7 +15,6 @@ import (
 	"slices"
 	"strings"
 
-	"github.com/google/go-github/v68/github"
 	libgithub "github.com/google/go-github/v68/github"
 	"github.com/hashicorp/vault/tools/pipeline/internal/pkg/changed"
 	libgit "github.com/hashicorp/vault/tools/pipeline/internal/pkg/git"
@@ -295,20 +294,12 @@ func (r *CreateBackportReq) Run(
 			res = &CreateBackportRes{}
 		}
 
-		// Figure out the comment body. Worst case it ought to be whatever error
-		// we've returned.
-		var body string
-		if res.Error != nil {
-			body = res.Error.Error()
-		}
-
 		// Set any known errors on the response before we create a comment, as the
 		// error will be used in the comment body if present.
 		res.Error = errors.Join(res.Error, os.Chdir(initialDir))
-		body = res.CommentBody()
 		var err1 error
 		res.Comment, err1 = createPullRequestComment(
-			ctx, github, r.Owner, r.Repo, int(r.PullNumber), body,
+			ctx, github, r.Owner, r.Repo, int(r.PullNumber), res.CommentBody(),
 		)
 
 		// Set our finalized error on our response and also update our returned error
@@ -589,7 +580,7 @@ func (r *CreateBackportReq) backportRef(
 	ctx context.Context,
 	git *libgit.Client,
 	github *libgithub.Client,
-	pr *github.PullRequest,
+	pr *libgithub.PullRequest,
 	activeVersions map[string]*releases.Version,
 	changedFiles *ListChangedFilesRes,
 	ref string, // the full base ref of the branch we're backporting to
@@ -748,7 +739,7 @@ func (r *CreateBackportReq) backportRef(
 func (r *CreateBackportReq) backportCECommitWithPatch(
 	ctx context.Context,
 	git *libgit.Client,
-	pr *github.PullRequest,
+	pr *libgithub.PullRequest,
 	changedFiles *ListChangedFilesRes,
 	commitSHA string,
 ) error {
@@ -994,11 +985,7 @@ func (r *CreateBackportReq) hasEntPrefix(ref string) bool {
 // isEnt takes a branch reference and determines whether or not it refers to
 // an enterprise branch.
 func (r *CreateBackportReq) isEnt(ref string) bool {
-	if r.hasCEPrefix(ref) {
-		return false
-	}
-
-	return true
+	return !r.hasCEPrefix(ref)
 }
 
 // shouldSkipRef determines whether or we ought to backport to a given branch

tools/pipeline/internal/pkg/github/list_workflow_runs_test.go

@@ -28,7 +28,7 @@ func TestWorkflowRunSummaryTemplate(t *testing.T) {
 		require.NotNil(t, run)
 		run.summary = ""
 		summary, err := run.Summary()
-		require.NoError(t, err)
+		require.NoErrorf(t, err, summary)
 		require.NotEmpty(t, summary)
 		// t.Log(summary) // useful to see rendered output when modifying
 	}

tools/pipeline/internal/pkg/github/sync_branch_request.go

@@ -0,0 +1,233 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package github
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+
+	libgithub "github.com/google/go-github/v68/github"
+	libgit "github.com/hashicorp/vault/tools/pipeline/internal/pkg/git"
+	"github.com/jedib0t/go-pretty/v6/table"
+	slogctx "github.com/veqryn/slog-context"
+)
+
+// SyncBranchReq is a request to synchronize two github hosted branches with
+// a git merge from one into another.
+//
+// NOTE: We require that both branches exist for the operation to succeed.
+type SyncBranchReq struct {
+	FromOwner  string
+	FromRepo   string
+	FromOrigin string
+	FromBranch string
+	ToOwner    string
+	ToRepo     string
+	ToOrigin   string
+	ToBranch   string
+	RepoDir    string
+}
+
+// SyncBranchRes is a copy pull request response.
+type SyncBranchRes struct {
+	Error   error          `json:"error,omitempty"`
+	Request *SyncBranchReq `json:"request,omitempty"`
+}
+
+// Run runs the request to synchronize a branches via a merge.
+func (r *SyncBranchReq) Run(
+	ctx context.Context,
+	github *libgithub.Client,
+	git *libgit.Client,
+) (*SyncBranchRes, error) {
+	var err error
+	res := &SyncBranchRes{Request: r}
+
+	slog.Default().DebugContext(slogctx.Append(ctx,
+		slog.String("from-owner", r.FromOwner),
+		slog.String("from-repo", r.FromRepo),
+		slog.String("from-origin", r.FromOrigin),
+		slog.String("from-branch", r.FromBranch),
+		slog.String("to-owner", r.ToOwner),
+		slog.String("to-repo", r.ToRepo),
+		slog.String("to-origin", r.ToOrigin),
+		slog.String("to-branch", r.ToBranch),
+		slog.String("to-owner", r.ToOwner),
+		slog.String("repo-dir", r.RepoDir),
+	), "synchronizing branches")
+
+	// Make sure we have required and valid fields
+	err = r.Validate(ctx)
+	if err != nil {
+		return res, err
+	}
+
+	// Make sure we've been given a valid location for a repo and/or create a
+	// temporary one
+	var tmpDir bool
+	r.RepoDir, err, tmpDir = ensureGitRepoDir(ctx, r.RepoDir)
+	if err != nil {
+		return res, err
+	}
+	if tmpDir {
+		defer os.RemoveAll(r.RepoDir)
+	}
+
+	// Clone the remote repository and fetch the branch we're going to merge into.
+	// These will change our working directory into RepoDir.
+	_, err = os.Stat(filepath.Join(r.RepoDir, ".git"))
+	if err == nil {
+		err = initializeExistingRepo(
+			ctx, git, r.RepoDir, r.ToOrigin, r.ToBranch,
+		)
+	} else {
+		err = initializeNewRepo(
+			ctx, git, r.RepoDir, r.ToOwner, r.ToRepo, r.ToOrigin, r.ToBranch,
+		)
+	}
+	if err != nil {
+		return res, err
+	}
+
+	// Check out our branch. Our intialization above will ensure we have a local
+	// reference.
+	slog.Default().DebugContext(ctx, "checking out to-branch")
+	checkoutRes, err := git.Checkout(ctx, &libgit.CheckoutOpts{
+		Branch: r.ToBranch,
+	})
+	if err != nil {
+		return res, fmt.Errorf("checking out to-branch: %s: %w", checkoutRes.String(), err)
+	}
+
+	// Add our from upstream as a remote and fetch our from branch.
+	slog.Default().DebugContext(ctx, "adding from upstream and fetching from-branch")
+	remoteRes, err := git.Remote(ctx, &libgit.RemoteOpts{
+		Command: libgit.RemoteCommandAdd,
+		Track:   []string{r.FromBranch},
+		Fetch:   true,
+		Name:    r.FromOrigin,
+		URL:     fmt.Sprintf("https://github.com/%s/%s.git", r.FromOwner, r.FromRepo),
+	})
+	if err != nil {
+		err = fmt.Errorf("fetching from branch: %s, %w", remoteRes.String(), err)
+		return res, err
+	}
+
+	// Use our remote reference as we haven't created a local reference.
+	fromBranch := "remotes/" + r.FromOrigin + "/" + r.FromBranch
+	slog.Default().DebugContext(ctx, "merging from-branch into to-branch")
+	mergeRes, err := git.Merge(ctx, &libgit.MergeOpts{
+		NoVerify: true,
+		Strategy: libgit.MergeStrategyORT,
+		StrategyOptions: []libgit.MergeStrategyOption{
+			libgit.MergeStrategyOptionTheirs,
+			libgit.MergeStrategyOptionIgnoreSpaceChange,
+		},
+		IntoName: r.ToBranch,
+		Commit:   fromBranch,
+	})
+	if err != nil {
+		return res, fmt.Errorf("merging from-branch into to-branch: %s: %w", mergeRes.String(), err)
+	}
+
+	slog.Default().DebugContext(ctx, "pushing to-branch")
+	pushRes, err := git.Push(ctx, &libgit.PushOpts{
+		Repository: r.ToOrigin,
+		Refspec:    []string{r.ToBranch},
+	})
+	if err != nil {
+		return res, fmt.Errorf("pushing to-branch: %s: %w", pushRes.String(), err)
+	}
+
+	return res, nil
+}
+
+// validate ensures that we've been given the minimum filter arguments necessary to complete a
+// request. It is always recommended that additional fitlers be given to reduce the response size
+// and not exhaust API limits.
+func (r *SyncBranchReq) Validate(ctx context.Context) error {
+	// TODO
+	if r == nil {
+		return errors.New("failed to initialize request")
+	}
+
+	if r.FromOrigin == "" {
+		return errors.New("no github from origin has been provided")
+	}
+
+	if r.FromOwner == "" {
+		return errors.New("no github from owner has been provided")
+	}
+
+	if r.FromRepo == "" {
+		return errors.New("no github from repository has been provided")
+	}
+
+	if r.FromBranch == "" {
+		return errors.New("no github from branch has been provided")
+	}
+
+	if r.ToOrigin == "" {
+		return errors.New("no github to origin has been provided")
+	}
+
+	if r.ToOwner == "" {
+		return errors.New("no github to owner has been provided")
+	}
+
+	if r.ToRepo == "" {
+		return errors.New("no github to repository has been provided")
+	}
+
+	if r.ToBranch == "" {
+		return errors.New("no github to branch has been provided")
+	}
+
+	return nil
+}
+
+// ToJSON marshals the response to JSON.
+func (r *SyncBranchRes) ToJSON() ([]byte, error) {
+	b, err := json.Marshal(r)
+	if err != nil {
+		return nil, fmt.Errorf("marshaling list changed files to JSON: %w", err)
+	}
+
+	return b, nil
+}
+
+// ToTable marshals the response to a text table.
+func (r *SyncBranchRes) ToTable(err error) table.Writer {
+	t := table.NewWriter()
+	t.Style().Options.DrawBorder = false
+	t.Style().Options.SeparateColumns = false
+	t.Style().Options.SeparateFooter = false
+	t.Style().Options.SeparateHeader = false
+	t.Style().Options.SeparateRows = false
+	t.AppendHeader(table.Row{
+		"From", "To", "Error",
+	})
+
+	if r.Request != nil {
+		from := r.Request.FromOwner + "/" + r.Request.FromRepo + "/" + r.Request.FromBranch
+		to := r.Request.ToOwner + "/" + r.Request.ToRepo + "/" + r.Request.ToBranch
+		row := table.Row{from, to}
+		if err != nil {
+			row = append(row, err.Error())
+		} else {
+			row = append(row, nil)
+		}
+		t.AppendRow(row)
+	}
+
+	t.SuppressEmptyColumns()
+	t.SuppressTrailingSpaces()
+
+	return t
+}

tools/pipeline/internal/pkg/github/templates_test.go

@@ -158,6 +158,7 @@ func Test_renderEmbeddedTemplateToTmpFile_copyPRComment(t *testing.T) {
 			require.NoError(t, err)
 			defer file.Close()
 			bytes, err := io.ReadAll(file)
+			require.NoError(t, err)
 			require.NotEmpty(t, bytes)
 			for _, c := range test.coAuthoredByTrailers {
 				require.Contains(t, string(bytes), c)

tools/pipeline/internal/pkg/github/workflow_run_summary.go

@@ -13,7 +13,7 @@ import (
 
 // workflowRunTemplate is our template for rendering workflow runs in human
 // readable text.
-var workflowRunTemplate = template.Must(template.New("workflow_run").Funcs(template.FuncMap{
+var workflowRunTemplate = template.Must(template.New("workflow-run-text.tmpl").Funcs(template.FuncMap{
 	"boldify":              boldify,
 	"format_log_lines":     formatLogLines,
 	"intensify_status":     intensifyStatus,