diff --git a/tools/pipeline/internal/cmd/github.go b/tools/pipeline/internal/cmd/github.go index 830c69f660..a2c4008a24 100644 --- a/tools/pipeline/internal/cmd/github.go +++ b/tools/pipeline/internal/cmd/github.go @@ -40,6 +40,7 @@ func newGithubCmd() *cobra.Command { githubCmd.AddCommand(newGithubCopyCmd()) githubCmd.AddCommand(newGithubCreateCmd()) githubCmd.AddCommand(newGithubListCmd()) + githubCmd.AddCommand(newGithubSyncCmd()) return githubCmd } diff --git a/tools/pipeline/internal/cmd/github_copy_pull_request.go b/tools/pipeline/internal/cmd/github_copy_pull_request.go index d7b5f3f922..e009c74d93 100644 --- a/tools/pipeline/internal/cmd/github_copy_pull_request.go +++ b/tools/pipeline/internal/cmd/github_copy_pull_request.go @@ -68,7 +68,7 @@ func runCopyGithubPullRequestCmd(cmd *cobra.Command, args []string) error { } fmt.Println(string(b)) default: - fmt.Println(res.ToTable(err)) + fmt.Println(res.ToTable(err).Render()) } return err diff --git a/tools/pipeline/internal/cmd/github_sync.go b/tools/pipeline/internal/cmd/github_sync.go new file mode 100644 index 0000000000..8063b434b4 --- /dev/null +++ b/tools/pipeline/internal/cmd/github_sync.go @@ -0,0 +1,19 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package cmd + +import ( + "github.com/spf13/cobra" +) + +func newGithubSyncCmd() *cobra.Command { + syncCmd := &cobra.Command{ + Use: "sync", + Short: "Github sync commands", + Long: "Github sync commands", + } + syncCmd.AddCommand(newSyncGithubBranchCmd()) + + return syncCmd +} diff --git a/tools/pipeline/internal/cmd/github_sync_branch.go b/tools/pipeline/internal/cmd/github_sync_branch.go new file mode 100644 index 0000000000..65096d1eae --- /dev/null +++ b/tools/pipeline/internal/cmd/github_sync_branch.go @@ -0,0 +1,64 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package cmd + +import ( + "context" + "errors" + "fmt" + + "github.com/hashicorp/vault/tools/pipeline/internal/pkg/github" + "github.com/spf13/cobra" +) + +var syncGithubBranchReq = github.SyncBranchReq{} + +func newSyncGithubBranchCmd() *cobra.Command { + syncBranchCmd := &cobra.Command{ + Use: "branch", + Short: "Sync a branch between two repositories", + Long: "Sync a branch between two repositories by merging the --from branch into the --to branch and pushing the result on success", + RunE: runSyncGithubBranchCmd, + } + + syncBranchCmd.PersistentFlags().StringVar(&syncGithubBranchReq.FromOrigin, "from-origin", "from", "The origin name to use for the from-branch") + syncBranchCmd.PersistentFlags().StringVar(&syncGithubBranchReq.FromOwner, "from-owner", "hashicorp", "The Github organization hosting the from branch") + syncBranchCmd.PersistentFlags().StringVar(&syncGithubBranchReq.FromRepo, "from-repo", "vault-enterprise", "The Github repository to sync from") + syncBranchCmd.PersistentFlags().StringVar(&syncGithubBranchReq.FromBranch, "from-branch", "", "The name of the branch we want to sync from") + syncBranchCmd.PersistentFlags().StringVar(&syncGithubBranchReq.ToOrigin, "to-origin", "to", "The origin name to use for the to-branch") + syncBranchCmd.PersistentFlags().StringVar(&syncGithubBranchReq.ToOwner, "to-owner", "hashicorp", "The Github organization hosting the to branch") + syncBranchCmd.PersistentFlags().StringVar(&syncGithubBranchReq.ToRepo, "to-repo", "vault-enterprise", "The Github repository to sync to") + syncBranchCmd.PersistentFlags().StringVar(&syncGithubBranchReq.ToBranch, "to-branch", "", "The name of the branch we want to sync to") + syncBranchCmd.PersistentFlags().StringVarP(&syncGithubBranchReq.RepoDir, "repo-dir", "d", "", "The path to the vault repository dir. If not set a temporary directory will be used") + + err := syncBranchCmd.MarkPersistentFlagRequired("from-branch") + if err != nil { + panic(err) + } + err = syncBranchCmd.MarkPersistentFlagRequired("to-branch") + if err != nil { + panic(err) + } + + return syncBranchCmd +} + +func runSyncGithubBranchCmd(cmd *cobra.Command, args []string) error { + cmd.SilenceUsage = true // Don't spam the usage on failure + + res, err := syncGithubBranchReq.Run(context.TODO(), githubCmdState.Github, githubCmdState.Git) + + switch rootCfg.format { + case "json": + b, err1 := res.ToJSON() + if err1 != nil { + return errors.Join(err, err1) + } + fmt.Println(string(b)) + default: + fmt.Println(res.ToTable(err).Render()) + } + + return err +} diff --git a/tools/pipeline/internal/pkg/git/client.go b/tools/pipeline/internal/pkg/git/client.go index a0bc2cf1b2..528043b520 100644 --- a/tools/pipeline/internal/pkg/git/client.go +++ b/tools/pipeline/internal/pkg/git/client.go @@ -12,7 +12,6 @@ import ( "net/url" "os" "os/exec" - oexec "os/exec" "strings" "sync" @@ -99,7 +98,7 @@ func (c *Client) Exec(ctx context.Context, subCmd string, opts OptStringer) (*Ex env = append(env, res.Env...) } - cmd := oexec.Command("git", append([]string{subCmd}, opts.Strings()...)...) + cmd := exec.Command("git", append([]string{subCmd}, opts.Strings()...)...) cmd.Env = env res.Cmd = cmd.String() ctx = slogctx.Append(ctx, slog.String("cmd", cmd.String())) diff --git a/tools/pipeline/internal/pkg/github/copy_pull_request.go b/tools/pipeline/internal/pkg/github/copy_pull_request.go index f10b80d22b..1303c24d22 100644 --- a/tools/pipeline/internal/pkg/github/copy_pull_request.go +++ b/tools/pipeline/internal/pkg/github/copy_pull_request.go @@ -78,17 +78,9 @@ func (r *CopyPullRequestReq) Run( res = &CopyPullRequestRes{Request: r} } - // Figure out the comment body. Worst case it ought to be whatever error - // we've returned. - var body string - if err != nil { - body = err.Error() - } - // Set any known errors on the response before we create a comment, as the // error will be used in the comment body if present. err = errors.Join(err, os.Chdir(initialDir)) - body = res.CommentBody(err) var err1 error res.Comment, err1 = createPullRequestComment( ctx, @@ -96,7 +88,7 @@ func (r *CopyPullRequestReq) Run( r.FromOwner, r.FromRepo, int(r.PullNumber), - body, + res.CommentBody(err), ) // Set our finalized error on our response and also update our returned error @@ -118,7 +110,7 @@ func (r *CopyPullRequestReq) Run( return res, err } if tmpDir { - // defer os.RemoveAll(r.RepoDir) + defer os.RemoveAll(r.RepoDir) } // Get our pull request details @@ -332,7 +324,7 @@ func (r *CopyPullRequestReq) Validate(ctx context.Context) error { } if r.FromRepo == "" { - return errors.New("no github repository has been provided") + return errors.New("no github from repository has been provided") } if r.ToOrigin == "" { diff --git a/tools/pipeline/internal/pkg/github/create_backport.go b/tools/pipeline/internal/pkg/github/create_backport.go index a3fa927679..3b2312d932 100644 --- a/tools/pipeline/internal/pkg/github/create_backport.go +++ b/tools/pipeline/internal/pkg/github/create_backport.go @@ -15,7 +15,6 @@ import ( "slices" "strings" - "github.com/google/go-github/v68/github" libgithub "github.com/google/go-github/v68/github" "github.com/hashicorp/vault/tools/pipeline/internal/pkg/changed" libgit "github.com/hashicorp/vault/tools/pipeline/internal/pkg/git" @@ -295,20 +294,12 @@ func (r *CreateBackportReq) Run( res = &CreateBackportRes{} } - // Figure out the comment body. Worst case it ought to be whatever error - // we've returned. - var body string - if res.Error != nil { - body = res.Error.Error() - } - // Set any known errors on the response before we create a comment, as the // error will be used in the comment body if present. res.Error = errors.Join(res.Error, os.Chdir(initialDir)) - body = res.CommentBody() var err1 error res.Comment, err1 = createPullRequestComment( - ctx, github, r.Owner, r.Repo, int(r.PullNumber), body, + ctx, github, r.Owner, r.Repo, int(r.PullNumber), res.CommentBody(), ) // Set our finalized error on our response and also update our returned error @@ -589,7 +580,7 @@ func (r *CreateBackportReq) backportRef( ctx context.Context, git *libgit.Client, github *libgithub.Client, - pr *github.PullRequest, + pr *libgithub.PullRequest, activeVersions map[string]*releases.Version, changedFiles *ListChangedFilesRes, ref string, // the full base ref of the branch we're backporting to @@ -748,7 +739,7 @@ func (r *CreateBackportReq) backportRef( func (r *CreateBackportReq) backportCECommitWithPatch( ctx context.Context, git *libgit.Client, - pr *github.PullRequest, + pr *libgithub.PullRequest, changedFiles *ListChangedFilesRes, commitSHA string, ) error { @@ -994,11 +985,7 @@ func (r *CreateBackportReq) hasEntPrefix(ref string) bool { // isEnt takes a branch reference and determines whether or not it refers to // an enterprise branch. func (r *CreateBackportReq) isEnt(ref string) bool { - if r.hasCEPrefix(ref) { - return false - } - - return true + return !r.hasCEPrefix(ref) } // shouldSkipRef determines whether or we ought to backport to a given branch diff --git a/tools/pipeline/internal/pkg/github/list_workflow_runs_test.go b/tools/pipeline/internal/pkg/github/list_workflow_runs_test.go index bb89cc533d..2320ce8a3b 100644 --- a/tools/pipeline/internal/pkg/github/list_workflow_runs_test.go +++ b/tools/pipeline/internal/pkg/github/list_workflow_runs_test.go @@ -28,7 +28,7 @@ func TestWorkflowRunSummaryTemplate(t *testing.T) { require.NotNil(t, run) run.summary = "" summary, err := run.Summary() - require.NoError(t, err) + require.NoErrorf(t, err, summary) require.NotEmpty(t, summary) // t.Log(summary) // useful to see rendered output when modifying } diff --git a/tools/pipeline/internal/pkg/github/sync_branch_request.go b/tools/pipeline/internal/pkg/github/sync_branch_request.go new file mode 100644 index 0000000000..eef6c398c4 --- /dev/null +++ b/tools/pipeline/internal/pkg/github/sync_branch_request.go @@ -0,0 +1,233 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package github + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + + libgithub "github.com/google/go-github/v68/github" + libgit "github.com/hashicorp/vault/tools/pipeline/internal/pkg/git" + "github.com/jedib0t/go-pretty/v6/table" + slogctx "github.com/veqryn/slog-context" +) + +// SyncBranchReq is a request to synchronize two github hosted branches with +// a git merge from one into another. +// +// NOTE: We require that both branches exist for the operation to succeed. +type SyncBranchReq struct { + FromOwner string + FromRepo string + FromOrigin string + FromBranch string + ToOwner string + ToRepo string + ToOrigin string + ToBranch string + RepoDir string +} + +// SyncBranchRes is a copy pull request response. +type SyncBranchRes struct { + Error error `json:"error,omitempty"` + Request *SyncBranchReq `json:"request,omitempty"` +} + +// Run runs the request to synchronize a branches via a merge. +func (r *SyncBranchReq) Run( + ctx context.Context, + github *libgithub.Client, + git *libgit.Client, +) (*SyncBranchRes, error) { + var err error + res := &SyncBranchRes{Request: r} + + slog.Default().DebugContext(slogctx.Append(ctx, + slog.String("from-owner", r.FromOwner), + slog.String("from-repo", r.FromRepo), + slog.String("from-origin", r.FromOrigin), + slog.String("from-branch", r.FromBranch), + slog.String("to-owner", r.ToOwner), + slog.String("to-repo", r.ToRepo), + slog.String("to-origin", r.ToOrigin), + slog.String("to-branch", r.ToBranch), + slog.String("to-owner", r.ToOwner), + slog.String("repo-dir", r.RepoDir), + ), "synchronizing branches") + + // Make sure we have required and valid fields + err = r.Validate(ctx) + if err != nil { + return res, err + } + + // Make sure we've been given a valid location for a repo and/or create a + // temporary one + var tmpDir bool + r.RepoDir, err, tmpDir = ensureGitRepoDir(ctx, r.RepoDir) + if err != nil { + return res, err + } + if tmpDir { + defer os.RemoveAll(r.RepoDir) + } + + // Clone the remote repository and fetch the branch we're going to merge into. + // These will change our working directory into RepoDir. + _, err = os.Stat(filepath.Join(r.RepoDir, ".git")) + if err == nil { + err = initializeExistingRepo( + ctx, git, r.RepoDir, r.ToOrigin, r.ToBranch, + ) + } else { + err = initializeNewRepo( + ctx, git, r.RepoDir, r.ToOwner, r.ToRepo, r.ToOrigin, r.ToBranch, + ) + } + if err != nil { + return res, err + } + + // Check out our branch. Our intialization above will ensure we have a local + // reference. + slog.Default().DebugContext(ctx, "checking out to-branch") + checkoutRes, err := git.Checkout(ctx, &libgit.CheckoutOpts{ + Branch: r.ToBranch, + }) + if err != nil { + return res, fmt.Errorf("checking out to-branch: %s: %w", checkoutRes.String(), err) + } + + // Add our from upstream as a remote and fetch our from branch. + slog.Default().DebugContext(ctx, "adding from upstream and fetching from-branch") + remoteRes, err := git.Remote(ctx, &libgit.RemoteOpts{ + Command: libgit.RemoteCommandAdd, + Track: []string{r.FromBranch}, + Fetch: true, + Name: r.FromOrigin, + URL: fmt.Sprintf("https://github.com/%s/%s.git", r.FromOwner, r.FromRepo), + }) + if err != nil { + err = fmt.Errorf("fetching from branch: %s, %w", remoteRes.String(), err) + return res, err + } + + // Use our remote reference as we haven't created a local reference. + fromBranch := "remotes/" + r.FromOrigin + "/" + r.FromBranch + slog.Default().DebugContext(ctx, "merging from-branch into to-branch") + mergeRes, err := git.Merge(ctx, &libgit.MergeOpts{ + NoVerify: true, + Strategy: libgit.MergeStrategyORT, + StrategyOptions: []libgit.MergeStrategyOption{ + libgit.MergeStrategyOptionTheirs, + libgit.MergeStrategyOptionIgnoreSpaceChange, + }, + IntoName: r.ToBranch, + Commit: fromBranch, + }) + if err != nil { + return res, fmt.Errorf("merging from-branch into to-branch: %s: %w", mergeRes.String(), err) + } + + slog.Default().DebugContext(ctx, "pushing to-branch") + pushRes, err := git.Push(ctx, &libgit.PushOpts{ + Repository: r.ToOrigin, + Refspec: []string{r.ToBranch}, + }) + if err != nil { + return res, fmt.Errorf("pushing to-branch: %s: %w", pushRes.String(), err) + } + + return res, nil +} + +// validate ensures that we've been given the minimum filter arguments necessary to complete a +// request. It is always recommended that additional fitlers be given to reduce the response size +// and not exhaust API limits. +func (r *SyncBranchReq) Validate(ctx context.Context) error { + // TODO + if r == nil { + return errors.New("failed to initialize request") + } + + if r.FromOrigin == "" { + return errors.New("no github from origin has been provided") + } + + if r.FromOwner == "" { + return errors.New("no github from owner has been provided") + } + + if r.FromRepo == "" { + return errors.New("no github from repository has been provided") + } + + if r.FromBranch == "" { + return errors.New("no github from branch has been provided") + } + + if r.ToOrigin == "" { + return errors.New("no github to origin has been provided") + } + + if r.ToOwner == "" { + return errors.New("no github to owner has been provided") + } + + if r.ToRepo == "" { + return errors.New("no github to repository has been provided") + } + + if r.ToBranch == "" { + return errors.New("no github to branch has been provided") + } + + return nil +} + +// ToJSON marshals the response to JSON. +func (r *SyncBranchRes) ToJSON() ([]byte, error) { + b, err := json.Marshal(r) + if err != nil { + return nil, fmt.Errorf("marshaling list changed files to JSON: %w", err) + } + + return b, nil +} + +// ToTable marshals the response to a text table. +func (r *SyncBranchRes) ToTable(err error) table.Writer { + t := table.NewWriter() + t.Style().Options.DrawBorder = false + t.Style().Options.SeparateColumns = false + t.Style().Options.SeparateFooter = false + t.Style().Options.SeparateHeader = false + t.Style().Options.SeparateRows = false + t.AppendHeader(table.Row{ + "From", "To", "Error", + }) + + if r.Request != nil { + from := r.Request.FromOwner + "/" + r.Request.FromRepo + "/" + r.Request.FromBranch + to := r.Request.ToOwner + "/" + r.Request.ToRepo + "/" + r.Request.ToBranch + row := table.Row{from, to} + if err != nil { + row = append(row, err.Error()) + } else { + row = append(row, nil) + } + t.AppendRow(row) + } + + t.SuppressEmptyColumns() + t.SuppressTrailingSpaces() + + return t +} diff --git a/tools/pipeline/internal/pkg/github/templates_test.go b/tools/pipeline/internal/pkg/github/templates_test.go index ab8bd29163..d1dcbaf3e7 100644 --- a/tools/pipeline/internal/pkg/github/templates_test.go +++ b/tools/pipeline/internal/pkg/github/templates_test.go @@ -158,6 +158,7 @@ func Test_renderEmbeddedTemplateToTmpFile_copyPRComment(t *testing.T) { require.NoError(t, err) defer file.Close() bytes, err := io.ReadAll(file) + require.NoError(t, err) require.NotEmpty(t, bytes) for _, c := range test.coAuthoredByTrailers { require.Contains(t, string(bytes), c) diff --git a/tools/pipeline/internal/pkg/github/workflow_run_summary.go b/tools/pipeline/internal/pkg/github/workflow_run_summary.go index ade451c7fe..fcf62a3b8a 100644 --- a/tools/pipeline/internal/pkg/github/workflow_run_summary.go +++ b/tools/pipeline/internal/pkg/github/workflow_run_summary.go @@ -13,7 +13,7 @@ import ( // workflowRunTemplate is our template for rendering workflow runs in human // readable text. -var workflowRunTemplate = template.Must(template.New("workflow_run").Funcs(template.FuncMap{ +var workflowRunTemplate = template.Must(template.New("workflow-run-text.tmpl").Funcs(template.FuncMap{ "boldify": boldify, "format_log_lines": formatLogLines, "intensify_status": intensifyStatus,