Backport [VAULT-41857] pipeline(find-artifact): add support for finding artifacts from branches into ce/main (#11971)

* [VAULT-41857] pipeline(find-artifact): add support for finding artifacts from branches (#11799)

Add support for finding matching workflow artifacts from branches rather than PRs. This allows us to trigger custom HCP image builds from a branch rather than an PR. It also enables us to build and test the HCP image on a scheduled nightly cadence, which we've also enabled.

As part of these changes I also added support for specifying which environment you want to test and threaded it through the cloud scenario now that there are multiple variants. We also make the testing workflow workflow_dispatch-able so that we can trigger HVD testing for any custom image in any environment without building a new image.

Signed-off-by: Ryan Cragun <me@ryan.ec>
Co-authored-by: Ryan Cragun <me@ryan.ec>

.github/workflows/build-hcp-image.yml

@@ -3,6 +3,8 @@ name: build-hcp-image
 on:
   workflow_call:
     inputs:
+      branch:
+        type: string
       pull-request:
         type: number
       create-azure-image:

.github/workflows/build.yml

@@ -395,17 +395,20 @@ jobs:
   hcp-image:
     if: |
       needs.setup.outputs.is-ent-branch == 'true' &&
-      needs.setup.outputs.workflow-trigger == 'pull_request' &&
-      (
-        contains(fromJSON(needs.setup.outputs.labels), 'hcp/build-image') ||
-        contains(fromJSON(needs.setup.outputs.labels), 'hcp/test')
+      needs.setup.outputs.workflow-trigger == 'schedule' ||
+      ( needs.setup.outputs.workflow-trigger == 'pull_request' &&
+        (
+          contains(fromJSON(needs.setup.outputs.labels), 'hcp/build-image') ||
+          contains(fromJSON(needs.setup.outputs.labels), 'hcp/test')
+        )
       )
     needs:
       - setup
       - artifacts-ent
     uses: ./.github/workflows/build-hcp-image.yml
     with:
-      pull-request: ${{ github.event.pull_request.number }}
+      pull-request: ${{ needs.setup.outputs.workflow-trigger == 'pull_request' && github.event.pull_request.number || '' }}
+      branch: ${{ needs.setup.outputs.workflow-trigger == 'schedule' && 'main' || '' }}
       create-aws-image: true
       create-azure-image: false
       hcp-environment: int
@@ -481,6 +484,7 @@ jobs:
       - hcp-image
     uses: ./.github/workflows/test-run-enos-scenario-cloud.yml
     with:
+      hcp-environment: int
       product-version: ${{ fromJSON(needs.hcp-image.outputs.image).product_version }}
 
   completed-successfully:

.github/workflows/test-run-enos-scenario-cloud.yml

@@ -4,6 +4,8 @@ name: enos-cloud
 on:
   workflow_call:
     inputs:
+      hcp-environment:
+        type: string
       product-version:
         type: string
 

tools/pipeline/go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/Masterminds/semver v1.5.0
 	github.com/PuerkitoBio/goquery v1.11.0
 	github.com/avast/retry-go/v4 v4.6.1
-	github.com/google/go-github/v74 v74.0.0
+	github.com/google/go-github/v81 v81.0.0
 	github.com/hashicorp/hcl/v2 v2.24.0
 	github.com/hashicorp/releases-api v0.2.3
 	github.com/jedib0t/go-pretty/v6 v6.6.8

tools/pipeline/go.sum

@@ -81,8 +81,8 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
-github.com/google/go-github/v74 v74.0.0 h1:yZcddTUn8DPbj11GxnMrNiAnXH14gNs559AsUpNpPgM=
-github.com/google/go-github/v74 v74.0.0/go.mod h1:ubn/YdyftV80VPSI26nSJvaEsTOnsjrxG3o9kJhcyak=
+github.com/google/go-github/v81 v81.0.0 h1:hTLugQRxSLD1Yei18fk4A5eYjOGLUBKAl/VCqOfFkZc=
+github.com/google/go-github/v81 v81.0.0/go.mod h1:upyjaybucIbBIuxgJS7YLOZGziyvvJ92WX6WEBNE3sM=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=

tools/pipeline/internal/cmd/github.go

@@ -9,7 +9,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/google/go-github/v74/github"
+	"github.com/google/go-github/v81/github"
 	"github.com/hashicorp/vault/tools/pipeline/internal/pkg/git"
 	"github.com/shurcooL/githubv4"
 	"github.com/spf13/cobra"

tools/pipeline/internal/cmd/github_find_workflow_artifact.go

@@ -15,7 +15,7 @@ var findWorkflowArtifact = &github.FindWorkflowArtifactReq{}
 
 func newGithubFindWorkflowArtifactCmd() *cobra.Command {
 	findWorkflowArtifactCmd := &cobra.Command{
-		Use:   "workflow-artifact [--pr 1234 --workflow build --pattern 'vault_[0-9]'",
+		Use:   "workflow-artifact [--pr 1234 | --branch main] [--workflow build --pattern 'vault_[0-9]' ]",
 		Short: "Find an artifact associated with a pull requests workflow run",
 		Long:  "Find an artifact associated with a pull requests workflow run",
 		RunE:  runFindGithubWorkflowArtifactCmd,
@@ -26,6 +26,7 @@ func newGithubFindWorkflowArtifactCmd() *cobra.Command {
 	findWorkflowArtifactCmd.PersistentFlags().StringVarP(&findWorkflowArtifact.Owner, "owner", "o", "hashicorp", "The Github organization")
 	findWorkflowArtifactCmd.PersistentFlags().StringVarP(&findWorkflowArtifact.Repo, "repo", "r", "vault", "The Github repository. Private repositories require auth via a GITHUB_TOKEN env var")
 	findWorkflowArtifactCmd.PersistentFlags().IntVarP(&findWorkflowArtifact.PullNumber, "pr", "p", 0, "The pull request to use as the trigger of the workflow")
+	findWorkflowArtifactCmd.PersistentFlags().StringVarP(&findWorkflowArtifact.Branch, "branch", "b", "", "The branch to use as the trigger of the workflow")
 	findWorkflowArtifactCmd.PersistentFlags().StringVarP(&findWorkflowArtifact.WorkflowName, "workflow", "w", "", "The name of the workflow the artifact will be associated with")
 	findWorkflowArtifactCmd.PersistentFlags().BoolVar(&findWorkflowArtifact.WriteToGithubOutput, "github-output", false, "Whether or not to write 'workflow-artifact' to $GITHUB_OUTPUT")
 

tools/pipeline/internal/pkg/changed/checkers_test.go

@@ -7,7 +7,7 @@ import (
 	"context"
 	"testing"
 
-	"github.com/google/go-github/v74/github"
+	"github.com/google/go-github/v81/github"
 	"github.com/stretchr/testify/require"
 )
 

tools/pipeline/internal/pkg/changed/file.go

@@ -8,7 +8,7 @@ import (
 	"slices"
 	"strings"
 
-	gh "github.com/google/go-github/v74/github"
+	gh "github.com/google/go-github/v81/github"
 )
 
 type (

tools/pipeline/internal/pkg/github/add_assignees.go

@@ -8,7 +8,7 @@ import (
 	"log/slog"
 	"slices"
 
-	libgithub "github.com/google/go-github/v74/github"
+	libgithub "github.com/google/go-github/v81/github"
 	slogctx "github.com/veqryn/slog-context"
 )
 

tools/pipeline/internal/pkg/github/check_commit_status.go

@@ -10,7 +10,7 @@ import (
 	"slices"
 	"strings"
 
-	libgithub "github.com/google/go-github/v74/github"
+	libgithub "github.com/google/go-github/v81/github"
 	"github.com/jedib0t/go-pretty/v6/table"
 )
 

tools/pipeline/internal/pkg/github/check_go_mod_diff_request.go

@@ -11,7 +11,7 @@ import (
 	"log/slog"
 	"os"
 
-	libgithub "github.com/google/go-github/v74/github"
+	libgithub "github.com/google/go-github/v81/github"
 	libgit "github.com/hashicorp/vault/tools/pipeline/internal/pkg/git"
 	"github.com/hashicorp/vault/tools/pipeline/internal/pkg/golang"
 	"github.com/jedib0t/go-pretty/v6/table"

tools/pipeline/internal/pkg/github/close_copied_origin_pull_request.go

@@ -12,7 +12,7 @@ import (
 	"slices"
 	"strings"
 
-	libgithub "github.com/google/go-github/v74/github"
+	libgithub "github.com/google/go-github/v81/github"
 	"github.com/jedib0t/go-pretty/v6/table"
 	"github.com/shurcooL/githubv4"
 	slogctx "github.com/veqryn/slog-context"

tools/pipeline/internal/pkg/github/commit.go

@@ -7,7 +7,7 @@ import (
 	"context"
 	"log/slog"
 
-	libgithub "github.com/google/go-github/v74/github"
+	libgithub "github.com/google/go-github/v81/github"
 	slogctx "github.com/veqryn/slog-context"
 )
 

tools/pipeline/internal/pkg/github/copy_pull_request.go

@@ -15,7 +15,7 @@ import (
 	"slices"
 	"strings"
 
-	libgithub "github.com/google/go-github/v74/github"
+	libgithub "github.com/google/go-github/v81/github"
 	libgit "github.com/hashicorp/vault/tools/pipeline/internal/pkg/git"
 	"github.com/jedib0t/go-pretty/v6/table"
 	slogctx "github.com/veqryn/slog-context"

tools/pipeline/internal/pkg/github/copy_pull_request_test.go

@@ -6,7 +6,7 @@ package github
 import (
 	"testing"
 
-	libgithub "github.com/google/go-github/v74/github"
+	libgithub "github.com/google/go-github/v81/github"
 	"github.com/stretchr/testify/require"
 )
 

tools/pipeline/internal/pkg/github/create_backport.go

@@ -15,7 +15,7 @@ import (
 	"slices"
 	"strings"
 
-	libgithub "github.com/google/go-github/v74/github"
+	libgithub "github.com/google/go-github/v81/github"
 	"github.com/hashicorp/vault/tools/pipeline/internal/pkg/changed"
 	libgit "github.com/hashicorp/vault/tools/pipeline/internal/pkg/git"
 	"github.com/hashicorp/vault/tools/pipeline/internal/pkg/releases"

tools/pipeline/internal/pkg/github/create_backport_test.go

@@ -8,7 +8,7 @@ import (
 	"errors"
 	"testing"
 
-	libgithub "github.com/google/go-github/v74/github"
+	libgithub "github.com/google/go-github/v81/github"
 	"github.com/hashicorp/vault/tools/pipeline/internal/pkg/changed"
 	"github.com/hashicorp/vault/tools/pipeline/internal/pkg/releases"
 	"github.com/stretchr/testify/require"

tools/pipeline/internal/pkg/github/find_workflow_artifact.go

@@ -9,11 +9,13 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"log/slog"
 	"regexp"
 	"slices"
 
-	gh "github.com/google/go-github/v74/github"
+	gh "github.com/google/go-github/v81/github"
 	"github.com/jedib0t/go-pretty/v6/table"
+	slogctx "github.com/veqryn/slog-context"
 )
 
 // FindWorkflowArtifactReq is a request to find an artifact associated with a
@@ -23,6 +25,7 @@ type FindWorkflowArtifactReq struct {
 	ArtifactPattern     string
 	Owner               string
 	PullNumber          int
+	Branch              string
 	Repo                string
 	WorkflowName        string
 	WriteToGithubOutput bool
@@ -60,21 +63,106 @@ func (r *FindWorkflowArtifactReq) Run(ctx context.Context, client *gh.Client) (*
 		return nil, fmt.Errorf("getting workflow: %w", err)
 	}
 
-	// Get the pull request we're searching
-	res.PR, err = getPullRequest(ctx, client, r.Owner, r.Repo, r.PullNumber)
-	if err != nil {
-		return nil, fmt.Errorf("getting pull request: %w", err)
+	// Define our matcher. It can either be an exact match from an given name or
+	// match a given pattern.
+	byNameOrPattern := func(art *gh.Artifact) bool {
+		// If we've been given a name locate it by that
+		if r.ArtifactName != "" {
+			if art.GetName() == r.ArtifactName {
+				return true
+			}
+		}
+		// Find it by regex
+		if r.compiledPattern.MatchString(art.GetName()) {
+			return true
+		}
+
+		return false
 	}
 
+	if r.PullNumber != 0 {
+		// We've been configured to search for an artifact in reference to a Pull
+		// Request. Get the details and then search the branch associated with it.
+		res.PR, err = getPullRequest(ctx, client, r.Owner, r.Repo, r.PullNumber)
+		if err != nil {
+			return nil, fmt.Errorf("getting pull request: %w", err)
+		}
+
+		res.Artifact, err = findWorkflowArtifact(
+			ctx,
+			client,
+			r.Owner,
+			r.Repo,
+			res.Workflow.GetID(),
+			res.PR.GetHead().GetRef(),
+			res.PR.GetHead().GetSHA(),
+			byNameOrPattern,
+		)
+
+		return res, err
+	}
+
+	// We've been configured with a branch. Get the last 5 commits and we'll
+	// we'll walk back until we hopefully find a workflow with a matching artifact.
+	// We attempt more than one commit because not all commits to either main
+	// or release branches are guaranteed to create build artifacts.
+
+	ctx = slogctx.Append(ctx,
+		slog.String("owner", r.Owner),
+		slog.String("repo", r.Repo),
+		slog.String("repo", r.Branch),
+	)
+	slog.Default().DebugContext(ctx, "getting list of commits")
+	commits, _, err := client.Repositories.ListCommits(ctx, r.Owner, r.Repo, &gh.CommitsListOptions{
+		SHA:         r.Branch,
+		ListOptions: gh.ListOptions{PerPage: 5},
+	})
+	if err != nil {
+		return nil, fmt.Errorf("getting list of commits: %w", err)
+	}
+
+	var innerErr error
+	for _, commit := range commits {
+		res.Artifact, innerErr = findWorkflowArtifact(
+			ctx,
+			client,
+			r.Owner,
+			r.Repo,
+			res.Workflow.GetID(),
+			r.Branch,
+			commit.GetSHA(),
+			byNameOrPattern,
+		)
+		if innerErr != nil {
+			err = errors.Join(err, innerErr)
+			continue
+		}
+
+		return res, nil
+	}
+
+	return nil, errors.Join(errors.New("unable to find artifact matching given criteria"), err)
+}
+
+func findWorkflowArtifact(
+	ctx context.Context,
+	client *gh.Client,
+	owner string,
+	repo string,
+	workflowID int64,
+	branch string,
+	sha string,
+	matcher func(*gh.Artifact) bool,
+) (*gh.Artifact, error) {
 	// Get the workflow runs associated with the workflow and the PR
 	opts := &gh.ListWorkflowRunsOptions{
-		Branch:              res.PR.GetHead().GetRef(),
+		Branch:              branch,
 		ExcludePullRequests: false,
-		HeadSHA:             res.PR.GetHead().GetSHA(),
+		HeadSHA:             sha,
 		ListOptions:         gh.ListOptions{PerPage: PerPageMax},
 		Status:              "success",
 	}
-	runs, err := getWorkflowRuns(ctx, client, r.Owner, r.Repo, res.Workflow.GetID(), opts)
+	runs, err := getWorkflowRuns(ctx, client, owner, repo, workflowID, opts)
 	if err != nil {
 		return nil, fmt.Errorf("getting workflow runs: %w", err)
 	}
@@ -92,26 +180,14 @@ func (r *FindWorkflowArtifactReq) Run(ctx context.Context, client *gh.Client) (*
 
 	var artifacts gh.ArtifactList
 	for _, run := range runs {
-		artifacts, err = getWorkflowRunArtifacts(ctx, client, r.Owner, r.Repo, *run.Run.ID)
+		artifacts, err = getWorkflowRunArtifacts(ctx, client, owner, repo, *run.Run.ID)
 		if err != nil {
 			return nil, fmt.Errorf("getting artifacts for workflow run %d: %w", *run.Run.ID, err)
 		}
 
 		for _, art := range artifacts.Artifacts {
-			// If we've been given a name locate it by that
-			if r.ArtifactName != "" {
-				if art.GetName() == r.ArtifactName {
-					res.Artifact = art
-
-					return res, nil
-				}
-			} else {
-				// Find it by regex
-				if r.compiledPattern.MatchString(art.GetName()) {
-					res.Artifact = art
-
-					return res, nil
-				}
+			if matcher(art) {
+				return art, nil
 			}
 		}
 	}
@@ -134,8 +210,8 @@ func (r *FindWorkflowArtifactReq) validate() error {
 		return errors.New("no github repository has been provided")
 	}
 
-	if r.PullNumber == 0 {
-		return errors.New("no github pull request number has been provided")
+	if r.PullNumber == 0 && r.Branch == "" {
+		return errors.New("no github pull request number or branch has been provided")
 	}
 
 	if r.WorkflowName == "" {

tools/pipeline/internal/pkg/github/issue.go

@@ -7,7 +7,7 @@ import (
 	"context"
 	"log/slog"
 
-	libgithub "github.com/google/go-github/v74/github"
+	libgithub "github.com/google/go-github/v81/github"
 	slogctx "github.com/veqryn/slog-context"
 )
 

tools/pipeline/internal/pkg/github/list_changed_files.go

@@ -10,7 +10,7 @@ import (
 	"fmt"
 	"strings"
 
-	gh "github.com/google/go-github/v74/github"
+	gh "github.com/google/go-github/v81/github"
 	"github.com/hashicorp/vault/tools/pipeline/internal/pkg/changed"
 	"github.com/jedib0t/go-pretty/v6/table"
 )

tools/pipeline/internal/pkg/github/list_commit_statuses.go

@@ -8,7 +8,7 @@ import (
 	"errors"
 	"fmt"
 
-	libgithub "github.com/google/go-github/v74/github"
+	libgithub "github.com/google/go-github/v81/github"
 	"github.com/jedib0t/go-pretty/v6/table"
 )
 

tools/pipeline/internal/pkg/github/list_workflow_runs.go

@@ -10,7 +10,7 @@ import (
 	"net/http"
 	"sync"
 
-	gh "github.com/google/go-github/v74/github"
+	gh "github.com/google/go-github/v81/github"
 )
 
 // PerPageMax is the maximum number of entities to request for enpoints that

tools/pipeline/internal/pkg/github/pull_request.go

@@ -8,7 +8,7 @@ import (
 	"fmt"
 	"log/slog"
 
-	libgithub "github.com/google/go-github/v74/github"
+	libgithub "github.com/google/go-github/v81/github"
 	"github.com/shurcooL/githubv4"
 	slogctx "github.com/veqryn/slog-context"
 )

tools/pipeline/internal/pkg/github/sync_branch_request.go

@@ -12,7 +12,7 @@ import (
 	"os"
 	"path/filepath"
 
-	libgithub "github.com/google/go-github/v74/github"
+	libgithub "github.com/google/go-github/v81/github"
 	libgit "github.com/hashicorp/vault/tools/pipeline/internal/pkg/git"
 	"github.com/jedib0t/go-pretty/v6/table"
 	slogctx "github.com/veqryn/slog-context"

tools/pipeline/internal/pkg/github/templates_test.go

@@ -8,7 +8,7 @@ import (
 	"io"
 	"testing"
 
-	libgithub "github.com/google/go-github/v74/github"
+	libgithub "github.com/google/go-github/v81/github"
 	"github.com/stretchr/testify/require"
 )
 

tools/pipeline/internal/pkg/github/workflows.go

@@ -8,7 +8,7 @@ import (
 	"fmt"
 	"log/slog"
 
-	gh "github.com/google/go-github/v74/github"
+	gh "github.com/google/go-github/v81/github"
 	slogctx "github.com/veqryn/slog-context"
 )