From 51f56b8536592af3145ce7fe2bcad33f05b4b6db Mon Sep 17 00:00:00 2001 From: Vault Automation Date: Thu, 4 Sep 2025 17:20:25 -0600 Subject: [PATCH] [VAULT-39158, VAULT-39159]pipeline: add support for building HVD images (#9012) (#9130) * [VAULT-39159]: pipeline: add support for querying HCP image service In order to facilitate testing Vault Enterprise directly in HCP we need tools to both request an image be built from a candidate build and to also wait for the image to be available in order to execute test scenarios with it. This PR adds a few new `pipeline` sub-commands that can will be used for this purpose. `pipeline github find workflow-artifact` can be used to find the path of an artifact that matches the given filter criteria. You'll need to provide a pull request number, workflow name, and either an exact artifact name or a pattern. When providing a pattern only the first match will be returned so make sure your regular expression is robust. `pipeline hcp get image` will return the image information for an HCP image. You will need to supply auth via the `HCP_USERNAME` and `HCP_PASSWORD` environment variables in order to query the image service. It also takes an enviroment flag so you can query the image service in different environments. `pipeline hcp wait image` is like `pipeline hcp get image` except that it will continue to retry for a given timeout and with a given delay between requests. In this way it can be used to wait for an image to be available. As part of this we also update our Go modules to the latest versions that are compatible. * [VAULT-39158]: actions(build-hcp-image): add workflow for building HCP images * copywrite: add missing headers * remove unused output * address feedback * allow prerelease artifacts --------- Signed-off-by: Ryan Cragun Co-authored-by: Ryan Cragun --- tools/pipeline/go.mod | 44 ++-- tools/pipeline/go.sum | 109 ++++----- tools/pipeline/internal/cmd/github.go | 3 +- tools/pipeline/internal/cmd/github_find.go | 19 ++ .../cmd/github_find_workflow_artifact.go | 64 +++++ tools/pipeline/internal/cmd/hcp.go | 40 ++++ tools/pipeline/internal/cmd/hcp_show.go | 17 ++ tools/pipeline/internal/cmd/hcp_show_image.go | 64 +++++ tools/pipeline/internal/cmd/hcp_wait.go | 22 ++ tools/pipeline/internal/cmd/hcp_wait_image.go | 89 +++++++ tools/pipeline/internal/cmd/root.go | 5 +- .../internal/pkg/changed/checkers_test.go | 2 +- tools/pipeline/internal/pkg/changed/file.go | 2 +- .../internal/pkg/github/add_assignees.go | 2 +- .../internal/pkg/github/copy_pull_request.go | 2 +- .../pkg/github/copy_pull_request_test.go | 2 +- .../internal/pkg/github/create_backport.go | 2 +- .../pkg/github/create_backport_test.go | 2 +- .../pkg/github/find_workflow_artifact.go | 200 ++++++++++++++++ .../internal/pkg/github/list_changed_files.go | 2 +- .../internal/pkg/github/list_workflow_runs.go | 45 +--- .../internal/pkg/github/pull_request.go | 2 +- .../pkg/github/sync_branch_request.go | 2 +- .../internal/pkg/github/templates_test.go | 2 +- .../pipeline/internal/pkg/github/workflows.go | 119 ++++++++++ tools/pipeline/internal/pkg/hcp/client.go | 140 +++++++++++ .../pkg/hcp/get_latest_product_version.go | 221 ++++++++++++++++++ .../hcp/get_latest_product_version_test.go | 83 +++++++ .../internal/pkg/hcp/wait_for_image.go | 73 ++++++ 29 files changed, 1249 insertions(+), 130 deletions(-) create mode 100644 tools/pipeline/internal/cmd/github_find.go create mode 100644 tools/pipeline/internal/cmd/github_find_workflow_artifact.go create mode 100644 tools/pipeline/internal/cmd/hcp.go create mode 100644 tools/pipeline/internal/cmd/hcp_show.go create mode 100644 tools/pipeline/internal/cmd/hcp_show_image.go create mode 100644 tools/pipeline/internal/cmd/hcp_wait.go create mode 100644 tools/pipeline/internal/cmd/hcp_wait_image.go create mode 100644 tools/pipeline/internal/pkg/github/find_workflow_artifact.go create mode 100644 tools/pipeline/internal/pkg/github/workflows.go create mode 100644 tools/pipeline/internal/pkg/hcp/client.go create mode 100644 tools/pipeline/internal/pkg/hcp/get_latest_product_version.go create mode 100644 tools/pipeline/internal/pkg/hcp/get_latest_product_version_test.go create mode 100644 tools/pipeline/internal/pkg/hcp/wait_for_image.go diff --git a/tools/pipeline/go.mod b/tools/pipeline/go.mod index ee54a3c7a4..8db1d688c2 100644 --- a/tools/pipeline/go.mod +++ b/tools/pipeline/go.mod @@ -4,14 +4,15 @@ go 1.23.2 require ( github.com/Masterminds/semver v1.5.0 - github.com/google/go-github/v68 v68.0.0 - github.com/hashicorp/hcl/v2 v2.23.0 - github.com/hashicorp/releases-api v0.2.1 - github.com/jedib0t/go-pretty/v6 v6.6.7 + github.com/avast/retry-go/v4 v4.6.1 + github.com/google/go-github/v74 v74.0.0 + github.com/hashicorp/hcl/v2 v2.24.0 + github.com/hashicorp/releases-api v0.2.3 + github.com/jedib0t/go-pretty/v6 v6.6.8 github.com/spf13/cobra v1.9.1 github.com/stretchr/testify v1.10.0 - github.com/veqryn/slog-context v0.7.0 - github.com/zclconf/go-cty v1.16.2 + github.com/veqryn/slog-context v0.8.0 + github.com/zclconf/go-cty v1.16.4 ) require ( @@ -21,11 +22,11 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/docker/go-units v0.5.0 // indirect github.com/fatih/color v1.18.0 // indirect - github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/analysis v0.23.0 // indirect - github.com/go-openapi/errors v0.22.1 // indirect - github.com/go-openapi/jsonpointer v0.21.1 // indirect + github.com/go-openapi/errors v0.22.2 // indirect + github.com/go-openapi/jsonpointer v0.21.2 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/loads v0.22.0 // indirect github.com/go-openapi/runtime v0.28.0 // indirect @@ -52,21 +53,22 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/pointerstructure v1.2.1 // indirect github.com/oklog/ulid v1.3.1 // indirect - github.com/oklog/ulid/v2 v2.1.0 // indirect + github.com/oklog/ulid/v2 v2.1.1 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rivo/uniseg v0.4.7 // indirect - github.com/spf13/pflag v1.0.6 // indirect - go.mongodb.org/mongo-driver v1.17.3 // indirect + github.com/spf13/pflag v1.0.7 // indirect + go.mongodb.org/mongo-driver v1.17.4 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/otel v1.35.0 // indirect - go.opentelemetry.io/otel/metric v1.35.0 // indirect - go.opentelemetry.io/otel/trace v1.35.0 // indirect - golang.org/x/mod v0.24.0 // indirect - golang.org/x/net v0.38.0 // indirect - golang.org/x/sync v0.12.0 // indirect - golang.org/x/sys v0.31.0 // indirect - golang.org/x/text v0.23.0 // indirect - golang.org/x/tools v0.31.0 // indirect + go.opentelemetry.io/otel v1.37.0 // indirect + go.opentelemetry.io/otel/metric v1.37.0 // indirect + go.opentelemetry.io/otel/trace v1.37.0 // indirect + golang.org/x/mod v0.27.0 // indirect + golang.org/x/net v0.43.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/sys v0.35.0 // indirect + golang.org/x/text v0.28.0 // indirect + golang.org/x/tools v0.36.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/tools/pipeline/go.sum b/tools/pipeline/go.sum index 041d27b393..759ad46f75 100644 --- a/tools/pipeline/go.sum +++ b/tools/pipeline/go.sum @@ -22,8 +22,10 @@ github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= -github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/avast/retry-go/v4 v4.6.1 h1:VkOLRubHdisGrHnTu89g08aQEWEgRU7LVEop3GbIcMk= +github.com/avast/retry-go/v4 v4.6.1/go.mod h1:V6oF8njAwxJ5gRo1Q7Cxab24xs5NCWZBeaHHBklR8mA= +github.com/aws/aws-sdk-go v1.55.6 h1:cSg4pvZ3m8dgYcgqB97MrcdjUmZ1BeMYKUxMMB89IPk= +github.com/aws/aws-sdk-go v1.55.6/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= @@ -43,16 +45,16 @@ github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/ github.com/go-jose/go-jose/v3 v3.0.4 h1:Wp5HA7bLQcKnf6YYao/4kpRpVMp/yf6+pJKV8WFSaNY= github.com/go-jose/go-jose/v3 v3.0.4/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/analysis v0.23.0 h1:aGday7OWupfMs+LbmLZG4k0MYXIANxcuBTYUC03zFCU= github.com/go-openapi/analysis v0.23.0/go.mod h1:9mz9ZWaSlV8TvjQHLl2mUW2PbZtemkE8yA5v22ohupo= -github.com/go-openapi/errors v0.22.1 h1:kslMRRnK7NCb/CvR1q1VWuEQCEIsBGn5GgKD9e+HYhU= -github.com/go-openapi/errors v0.22.1/go.mod h1:+n/5UdIqdVnLIJ6Q9Se8HNGUXYaY6CN8ImWzfi/Gzp0= -github.com/go-openapi/jsonpointer v0.21.1 h1:whnzv/pNXtK2FbX/W9yJfRmE2gsmkfahjMKB0fZvcic= -github.com/go-openapi/jsonpointer v0.21.1/go.mod h1:50I1STOfbY1ycR8jGz8DaMeLCdXiI6aDteEdRNNzpdk= +github.com/go-openapi/errors v0.22.2 h1:rdxhzcBUazEcGccKqbY1Y7NS8FDcMyIRr0934jrYnZg= +github.com/go-openapi/errors v0.22.2/go.mod h1:+n/5UdIqdVnLIJ6Q9Se8HNGUXYaY6CN8ImWzfi/Gzp0= +github.com/go-openapi/jsonpointer v0.21.2 h1:AqQaNADVwq/VnkCmQg6ogE+M3FOsKTytwges0JdwVuA= +github.com/go-openapi/jsonpointer v0.21.2/go.mod h1:50I1STOfbY1ycR8jGz8DaMeLCdXiI6aDteEdRNNzpdk= github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= github.com/go-openapi/loads v0.22.0 h1:ECPGd4jX1U6NApCGG1We+uEozOAvXvJSF4nnwHZ8Aco= @@ -74,8 +76,8 @@ github.com/golang-migrate/migrate/v4 v4.14.1/go.mod h1:l7Ks0Au6fYHuUIxUhQ0rcVX1u github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-github/v68 v68.0.0 h1:ZW57zeNZiXTdQ16qrDiZ0k6XucrxZ2CGmoTvcCyQG6s= -github.com/google/go-github/v68 v68.0.0/go.mod h1:K9HAUBovM2sLwM408A18h+wd9vqdLOEqTUCbnRIcx68= +github.com/google/go-github/v74 v74.0.0 h1:yZcddTUn8DPbj11GxnMrNiAnXH14gNs559AsUpNpPgM= +github.com/google/go-github/v74 v74.0.0/go.mod h1:ubn/YdyftV80VPSI26nSJvaEsTOnsjrxG3o9kJhcyak= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -90,10 +92,10 @@ github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/C github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/hcl/v2 v2.23.0 h1:Fphj1/gCylPxHutVSEOf2fBOh1VE4AuLV7+kbJf3qos= -github.com/hashicorp/hcl/v2 v2.23.0/go.mod h1:62ZYHrXgPoX8xBnzl8QzbWq4dyDsDtfCRgIq1rbJEvA= -github.com/hashicorp/releases-api v0.2.1 h1:c2I97uYmhjCHIYy4OxfTy406jlrm/CZAYPpbKwA0pMs= -github.com/hashicorp/releases-api v0.2.1/go.mod h1:2+TZWrbqji5O7NIiE/rC/8rjMCPC3oC11FNDx9A5kiM= +github.com/hashicorp/hcl/v2 v2.24.0 h1:2QJdZ454DSsYGoaE6QheQZjtKZSUs9Nh2izTWiwQxvE= +github.com/hashicorp/hcl/v2 v2.24.0/go.mod h1:oGoO1FIQYfn/AgyOhlg9qLC6/nOJPX3qGbkZpYAcqfM= +github.com/hashicorp/releases-api v0.2.3 h1:mwNR+lKgJtIyeSQXYGM86fZ0u8ed09v7NS2ePKmVvyc= +github.com/hashicorp/releases-api v0.2.3/go.mod h1:J8AiSwS1Qy/m/RmHskUGDu9YQRLKreBBswc6ZTY5/tI= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -102,14 +104,14 @@ github.com/jackc/pgerrcode v0.0.0-20201024163028-a0d42d470451 h1:WAvSpGf7MsFuzAt github.com/jackc/pgerrcode v0.0.0-20201024163028-a0d42d470451/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.5.4 h1:Xp2aQS8uXButQdnCMWNmvx6UysWQQC+u1EoizjguY+8= -github.com/jackc/pgx/v5 v5.5.4/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= -github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= -github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= -github.com/jedib0t/go-pretty/v6 v6.6.7 h1:m+LbHpm0aIAPLzLbMfn8dc3Ht8MW7lsSO4MPItz/Uuo= -github.com/jedib0t/go-pretty/v6 v6.6.7/go.mod h1:YwC5CE4fJ1HFUDeivSV1r//AmANFHyqczZk+U6BDALU= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= +github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jedib0t/go-pretty/v6 v6.6.8 h1:JnnzQeRz2bACBobIaa/r+nqjvws4yEhcmaZ4n1QzsEc= +github.com/jedib0t/go-pretty/v6 v6.6.8/go.mod h1:YwC5CE4fJ1HFUDeivSV1r//AmANFHyqczZk+U6BDALU= github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bBK4= github.com/jessevdk/go-flags v1.6.1/go.mod h1:Mk8T1hIAWpOiJiHa9rJASDK2UGWji0EuPGBnNLMooyc= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -147,8 +149,8 @@ github.com/mitchellh/pointerstructure v1.2.1 h1:ZhBBeX8tSlRpu/FFhXH4RC4OJzFlqsQh github.com/mitchellh/pointerstructure v1.2.1/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU= -github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ= +github.com/oklog/ulid/v2 v2.1.1 h1:suPZ4ARWLOJLegGFiZZ1dFAkqzhMjL3J1TzI+5wHz8s= +github.com/oklog/ulid/v2 v2.1.1/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/outcaste-io/ristretto v0.2.3 h1:AK4zt/fJ76kjlYObOeNwh4T3asEuaCmp26pOvUOL9w0= @@ -171,8 +173,9 @@ github.com/secure-systems-lab/go-securesystemslib v0.7.0 h1:OwvJ5jQf9LnIAS83waAj github.com/secure-systems-lab/go-securesystemslib v0.7.0/go.mod h1:/2gYnlnHVQ6xeGtfIqFy7Do03K4cdCY0A/GlJLDKLHI= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.7 h1:vN6T9TfwStFPFM5XzjsvmzZkLuaLX+HS+0SeFLRgU6M= +github.com/spf13/pflag v1.0.7/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= @@ -180,53 +183,53 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0= github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw= -github.com/veqryn/slog-context v0.7.0 h1:Ne7ajlR6Mjs2rQQtpg8k0eO6krR5wzpareh5VpV+V2s= -github.com/veqryn/slog-context v0.7.0/go.mod h1:E+qpdyiQs2YKRxFnX1JjpdFE1z3Ka94Kem2q9ZG6Jjo= -github.com/zclconf/go-cty v1.16.2 h1:LAJSwc3v81IRBZyUVQDUdZ7hs3SYs9jv0eZJDWHD/70= -github.com/zclconf/go-cty v1.16.2/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= +github.com/veqryn/slog-context v0.8.0 h1:lDhwAgjwx52K5StqqQzi5d0Y/F4SNyGZbsXGd8MtucM= +github.com/veqryn/slog-context v0.8.0/go.mod h1:8rsT72p0kzzN9lmkwtabIhxg7ZkpnKblt9x3Eix8Tc0= +github.com/zclconf/go-cty v1.16.4 h1:QGXaag7/7dCzb+odlGrgr+YmYZFaOCMW6DEpS+UD1eE= +github.com/zclconf/go-cty v1.16.4/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo= github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM= -go.mongodb.org/mongo-driver v1.17.3 h1:TQyXhnsWfWtgAhMtOgtYHMTkZIfBTpMTsMnd9ZBeHxQ= -go.mongodb.org/mongo-driver v1.17.3/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= +go.mongodb.org/mongo-driver v1.17.4 h1:jUorfmVzljjr0FLzYQsGP8cgN/qzzxlY9Vh0C9KFXVw= +go.mongodb.org/mongo-driver v1.17.4/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= -go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= -go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= -go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= -go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= -go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= -golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= -golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= +golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU= -golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/DataDog/dd-trace-go.v1 v1.66.0 h1:025+lLubGtpiDWrRmSOxoFBPIiVRVYRcqP9oLabVOeg= gopkg.in/DataDog/dd-trace-go.v1 v1.66.0/go.mod h1:Av6AXGmQCQAbDnwNoPiuUz1k3GS8TwQjj+vEdwmEpmM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/tools/pipeline/internal/cmd/github.go b/tools/pipeline/internal/cmd/github.go index a2c4008a24..0858c85904 100644 --- a/tools/pipeline/internal/cmd/github.go +++ b/tools/pipeline/internal/cmd/github.go @@ -8,7 +8,7 @@ import ( "os" "path/filepath" - "github.com/google/go-github/v68/github" + "github.com/google/go-github/v74/github" "github.com/hashicorp/vault/tools/pipeline/internal/pkg/git" "github.com/spf13/cobra" ) @@ -39,6 +39,7 @@ func newGithubCmd() *cobra.Command { } githubCmd.AddCommand(newGithubCopyCmd()) githubCmd.AddCommand(newGithubCreateCmd()) + githubCmd.AddCommand(newGithubFindCmd()) githubCmd.AddCommand(newGithubListCmd()) githubCmd.AddCommand(newGithubSyncCmd()) diff --git a/tools/pipeline/internal/cmd/github_find.go b/tools/pipeline/internal/cmd/github_find.go new file mode 100644 index 0000000000..9f949ed43b --- /dev/null +++ b/tools/pipeline/internal/cmd/github_find.go @@ -0,0 +1,19 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package cmd + +import ( + "github.com/spf13/cobra" +) + +func newGithubFindCmd() *cobra.Command { + findCmd := &cobra.Command{ + Use: "find", + Short: "Github find commands", + Long: "Github find commands", + } + findCmd.AddCommand(newGithubFindWorkflowArtifactCmd()) + + return findCmd +} diff --git a/tools/pipeline/internal/cmd/github_find_workflow_artifact.go b/tools/pipeline/internal/cmd/github_find_workflow_artifact.go new file mode 100644 index 0000000000..d9c2379982 --- /dev/null +++ b/tools/pipeline/internal/cmd/github_find_workflow_artifact.go @@ -0,0 +1,64 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package cmd + +import ( + "context" + "fmt" + + "github.com/hashicorp/vault/tools/pipeline/internal/pkg/github" + "github.com/spf13/cobra" +) + +var findWorkflowArtifact = &github.FindWorkflowArtifactReq{} + +func newGithubFindWorkflowArtifactCmd() *cobra.Command { + findWorkflowArtifactCmd := &cobra.Command{ + Use: "workflow-artifact [--pr 1234 --workflow build --pattern 'vault_[0-9]'", + Short: "Find an artifact associated with a pull requests workflow run", + Long: "Find an artifact associated with a pull requests workflow run", + RunE: runFindGithubWorkflowArtifactCmd, + } + + findWorkflowArtifactCmd.PersistentFlags().StringVarP(&findWorkflowArtifact.ArtifactName, "name", "n", "", "The exact artifact name to match") + findWorkflowArtifactCmd.PersistentFlags().StringVarP(&findWorkflowArtifact.ArtifactPattern, "pattern", "m", "", "A pattern to match an artifact. Only the first match will be returned") + findWorkflowArtifactCmd.PersistentFlags().StringVarP(&findWorkflowArtifact.Owner, "owner", "o", "hashicorp", "The Github organization") + findWorkflowArtifactCmd.PersistentFlags().StringVarP(&findWorkflowArtifact.Repo, "repo", "r", "vault", "The Github repository. Private repositories require auth via a GITHUB_TOKEN env var") + findWorkflowArtifactCmd.PersistentFlags().IntVarP(&findWorkflowArtifact.PullNumber, "pr", "p", 0, "The pull request to use as the trigger of the workflow") + findWorkflowArtifactCmd.PersistentFlags().StringVarP(&findWorkflowArtifact.WorkflowName, "workflow", "w", "", "The name of the workflow the artifact will be associated with") + findWorkflowArtifactCmd.PersistentFlags().BoolVar(&findWorkflowArtifact.WriteToGithubOutput, "github-output", false, "Whether or not to write 'workflow-artifact' to $GITHUB_OUTPUT") + + return findWorkflowArtifactCmd +} + +func runFindGithubWorkflowArtifactCmd(cmd *cobra.Command, args []string) error { + cmd.SilenceUsage = true // Don't spam the usage on failure + + res, err := findWorkflowArtifact.Run(context.TODO(), githubCmdState.Github) + if err != nil { + return fmt.Errorf("listing github workflow failures: %w", err) + } + + switch rootCfg.format { + case "json": + jsonBytes, err := res.ToJSON() + if err != nil { + return err + } + fmt.Println(string(jsonBytes)) + default: + fmt.Println(res.ToTable()) + } + + if findWorkflowArtifact.WriteToGithubOutput { + jsonBytes, err := res.ToGithubOutput() + if err != nil { + return err + } + + return writeToGithubOutput("workflow-artifact", jsonBytes) + } + + return nil +} diff --git a/tools/pipeline/internal/cmd/hcp.go b/tools/pipeline/internal/cmd/hcp.go new file mode 100644 index 0000000000..94a17b75e5 --- /dev/null +++ b/tools/pipeline/internal/cmd/hcp.go @@ -0,0 +1,40 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package cmd + +import ( + "fmt" + "os" + + "github.com/hashicorp/vault/tools/pipeline/internal/pkg/hcp" + "github.com/spf13/cobra" +) + +var hcpCmdState = &struct { + client *hcp.Client +}{} + +func newHCPCmd() *cobra.Command { + env := "" + hcpCmd := &cobra.Command{ + Use: "hcp", + Short: "HCP commands", + Long: "HCP commands", + PersistentPreRun: func(cmd *cobra.Command, args []string) { + hcpCmdState.client = hcp.NewClient( + hcp.WithEnvironment(hcp.Environment(env)), + hcp.WithLoadAuthFromEnv(), + ) + if _, set := os.LookupEnv("HCP_PASSWORD"); !set { + fmt.Println("\x1b[1;33;49mWARNING\x1b[0m: HCP_PASSWORD has not been set. You probably want to set it and HCP_USERNAME in order to authenticate with the image service") + } + }, + } + hcpCmd.AddCommand(newHCPShowCmd()) + hcpCmd.AddCommand(newHCPWaitCmd()) + + hcpCmd.PersistentFlags().StringVarP(&env, "environment", "e", "prod", "The HCP environment to use. E.g. dev, int, prod") + + return hcpCmd +} diff --git a/tools/pipeline/internal/cmd/hcp_show.go b/tools/pipeline/internal/cmd/hcp_show.go new file mode 100644 index 0000000000..3b8e2dc864 --- /dev/null +++ b/tools/pipeline/internal/cmd/hcp_show.go @@ -0,0 +1,17 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package cmd + +import "github.com/spf13/cobra" + +func newHCPShowCmd() *cobra.Command { + showCmd := &cobra.Command{ + Use: "show", + Short: "HCP show commands", + Long: "HCP show commands", + } + showCmd.AddCommand(newHCPShowImageCmd()) + + return showCmd +} diff --git a/tools/pipeline/internal/cmd/hcp_show_image.go b/tools/pipeline/internal/cmd/hcp_show_image.go new file mode 100644 index 0000000000..1b2a315afa --- /dev/null +++ b/tools/pipeline/internal/cmd/hcp_show_image.go @@ -0,0 +1,64 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package cmd + +import ( + "context" + "fmt" + + "github.com/hashicorp/vault/tools/pipeline/internal/pkg/hcp" + "github.com/spf13/cobra" +) + +var showHCPImageReq = &hcp.GetLatestProductVersionReq{} + +func newHCPShowImageCmd() *cobra.Command { + availability := "" + + showHCPImage := &cobra.Command{ + Use: "image", + Short: "Show details of an HCP image", + Long: "Show details of an HCP image", + PersistentPreRun: func(cmd *cobra.Command, args []string) { + showHCPImageReq.Availability = hcp.GetLatestProductVersionAvailability(availability) + }, + RunE: runHCPImageShowLatestCmd, + } + + showHCPImage.PersistentFlags().StringVarP(&showHCPImageReq.ProductName, "product-name", "p", "vault", "The product or component of the image") + showHCPImage.PersistentFlags().StringVarP(&showHCPImageReq.ProductVersionConstraint, "product-version-constraint", "v", "", "A comma seperated list of constraints. If left unset the latest will be returned") + showHCPImage.PersistentFlags().StringVarP(&showHCPImageReq.HostManagerVersionConstraint, "host-manager-version-constraint", "m", "", "A semver string. If left unset the latest will be used") + showHCPImage.PersistentFlags().StringVarP(&showHCPImageReq.CloudProvider, "cloud", "c", "aws", "The cloud provider you wish to search. E.g. aws, azure") + showHCPImage.PersistentFlags().StringVarP(&showHCPImageReq.CloudRegion, "region", "r", "us-west-2", "The cloud region you wish to search") + showHCPImage.PersistentFlags().StringVarP(&availability, "availability", "a", "public", "The image availability") + showHCPImage.PersistentFlags().BoolVarP(&showHCPImageReq.ExcludeReleaseCandidates, "exclude-release-candidates", "x", false, "Exclude release candidates") + + return showHCPImage +} + +func runHCPImageShowLatestCmd(cmd *cobra.Command, args []string) error { + cmd.SilenceUsage = true // Don't spam the usage on failure + + res, err := showHCPImageReq.Run(context.TODO(), hcpCmdState.client) + if err != nil { + return fmt.Errorf("showing HCP image: %w", err) + } + + switch rootCfg.format { + case "json": + b, err := res.ToJSON() + if err != nil { + return err + } + fmt.Println(string(b)) + case "markdown": + tbl := res.ToTable() + tbl.SetTitle("HCP Image") + fmt.Println(tbl.RenderMarkdown()) + default: + fmt.Println(res.ToTable().Render()) + } + + return nil +} diff --git a/tools/pipeline/internal/cmd/hcp_wait.go b/tools/pipeline/internal/cmd/hcp_wait.go new file mode 100644 index 0000000000..ec91922eab --- /dev/null +++ b/tools/pipeline/internal/cmd/hcp_wait.go @@ -0,0 +1,22 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package cmd + +import ( + "errors" + + "github.com/spf13/cobra" +) + +func newHCPWaitCmd() *cobra.Command { + waitCmd := &cobra.Command{ + Use: "wait", + Short: "HCP wait commands", + Long: "HCP wait commands", + RunE: func(*cobra.Command, []string) error { return errors.New("unimplemented") }, + } + waitCmd.AddCommand(newHCPWaitForImageCmd()) + + return waitCmd +} diff --git a/tools/pipeline/internal/cmd/hcp_wait_image.go b/tools/pipeline/internal/cmd/hcp_wait_image.go new file mode 100644 index 0000000000..8054ae5c00 --- /dev/null +++ b/tools/pipeline/internal/cmd/hcp_wait_image.go @@ -0,0 +1,89 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package cmd + +import ( + "context" + "errors" + "fmt" + "os" + "os/signal" + "syscall" + "time" + + "github.com/hashicorp/vault/tools/pipeline/internal/pkg/hcp" + "github.com/spf13/cobra" +) + +var waitForHCPImage = &hcp.WaitForImageReq{ + Req: &hcp.GetLatestProductVersionReq{}, +} + +func newHCPWaitForImageCmd() *cobra.Command { + availability := "" + var timeout time.Duration + + imageGetLatestCmd := &cobra.Command{ + Use: "image", + Short: "Show details of an HCP image", + Long: "Show details of an HCP image", + PersistentPreRun: func(cmd *cobra.Command, args []string) { + waitForHCPImage.Req.Availability = hcp.GetLatestProductVersionAvailability(availability) + }, + RunE: func(cmd *cobra.Command, args []string) error { + cmd.SilenceUsage = true // Don't spam the usage on failure + + ctx, cancelCause := context.WithCancelCause(context.Background()) + ctx, cancel := context.WithTimeoutCause(ctx, timeout, errors.New("timed out waiting for image")) + defer cancel() + + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + go func() { + select { + case <-ctx.Done(): + return + case s := <-c: + fmt.Printf("\x1b[1;33;49mWARNING\x1b[0m: received %s signal. Stopping now..\n", s) + cancelCause(fmt.Errorf("received signal %s", s)) + cancel() + } + }() + + res, err := waitForHCPImage.Run(ctx, hcpCmdState.client) + if err != nil { + return fmt.Errorf("waiting for an HCP image: %w", err) + } + + switch rootCfg.format { + case "json": + b, err := res.Res.ToJSON() + if err != nil { + return err + } + fmt.Println(string(b)) + case "markdown": + tbl := res.Res.ToTable() + tbl.SetTitle("HCP Image") + fmt.Println(tbl.RenderMarkdown()) + default: + fmt.Println(res.Res.ToTable().Render()) + } + + return nil + }, + } + + imageGetLatestCmd.PersistentFlags().StringVarP(&waitForHCPImage.Req.ProductName, "product-name", "p", "vault", "The product or component of the image") + imageGetLatestCmd.PersistentFlags().StringVarP(&waitForHCPImage.Req.ProductVersionConstraint, "product-version-constraint", "v", "", "A comma seperated list of constraints. If left unset the latest will be returned") + imageGetLatestCmd.PersistentFlags().StringVarP(&waitForHCPImage.Req.HostManagerVersionConstraint, "host-manager-version-constraint", "m", "", "A semver string. If left unset the latest will be used") + imageGetLatestCmd.PersistentFlags().StringVarP(&waitForHCPImage.Req.CloudProvider, "cloud", "c", "aws", "The cloud provider you wish to search. E.g. aws, azure") + imageGetLatestCmd.PersistentFlags().StringVarP(&waitForHCPImage.Req.CloudRegion, "region", "r", "us-west-2", "The cloud region you wish to search") + imageGetLatestCmd.PersistentFlags().StringVarP(&availability, "availability", "a", "public", "The image availability") + imageGetLatestCmd.PersistentFlags().BoolVarP(&waitForHCPImage.Req.ExcludeReleaseCandidates, "exclude-release-candidates", "x", false, "Exclude release candidates") + imageGetLatestCmd.PersistentFlags().DurationVarP(&waitForHCPImage.Delay, "delay", "d", 10*time.Second, "the time to wait in-between requests") + imageGetLatestCmd.PersistentFlags().DurationVarP(&timeout, "timeout", "t", 30*time.Minute, "the maximum duration to wait for the image") + + return imageGetLatestCmd +} diff --git a/tools/pipeline/internal/cmd/root.go b/tools/pipeline/internal/cmd/root.go index 0e34123be0..2df76e5a13 100644 --- a/tools/pipeline/internal/cmd/root.go +++ b/tools/pipeline/internal/cmd/root.go @@ -27,10 +27,11 @@ func newRootCmd() *cobra.Command { } rootCmd.PersistentFlags().StringVar(&rootCfg.logLevel, "log", "warn", "Set the log level. One of 'debug', 'info', 'warn', 'error'") - rootCmd.PersistentFlags().StringVarP(&rootCfg.format, "format", "f", "table", "The output format. Can be 'json' or 'table'") + rootCmd.PersistentFlags().StringVarP(&rootCfg.format, "format", "f", "table", "The output format. Can be 'json', 'table', and sometimes 'markdown'") rootCmd.AddCommand(newGenerateCmd()) rootCmd.AddCommand(newGithubCmd()) + rootCmd.AddCommand(newHCPCmd()) rootCmd.AddCommand(newReleasesCmd()) rootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error { @@ -51,7 +52,7 @@ func newRootCmd() *cobra.Command { slog.SetDefault(slog.New(h)) switch rootCfg.format { - case "json", "table": + case "json", "table", "markdown": default: return fmt.Errorf("unsupported format: %s", rootCfg.format) } diff --git a/tools/pipeline/internal/pkg/changed/checkers_test.go b/tools/pipeline/internal/pkg/changed/checkers_test.go index 931ac7afe8..bb6839f332 100644 --- a/tools/pipeline/internal/pkg/changed/checkers_test.go +++ b/tools/pipeline/internal/pkg/changed/checkers_test.go @@ -7,7 +7,7 @@ import ( "context" "testing" - "github.com/google/go-github/v68/github" + "github.com/google/go-github/v74/github" "github.com/stretchr/testify/require" ) diff --git a/tools/pipeline/internal/pkg/changed/file.go b/tools/pipeline/internal/pkg/changed/file.go index 447be1ef9e..d6ca529a30 100644 --- a/tools/pipeline/internal/pkg/changed/file.go +++ b/tools/pipeline/internal/pkg/changed/file.go @@ -7,7 +7,7 @@ import ( "slices" "strings" - gh "github.com/google/go-github/v68/github" + gh "github.com/google/go-github/v74/github" ) type ( diff --git a/tools/pipeline/internal/pkg/github/add_assignees.go b/tools/pipeline/internal/pkg/github/add_assignees.go index 24609341bd..c8039a917b 100644 --- a/tools/pipeline/internal/pkg/github/add_assignees.go +++ b/tools/pipeline/internal/pkg/github/add_assignees.go @@ -8,7 +8,7 @@ import ( "log/slog" "slices" - libgithub "github.com/google/go-github/v68/github" + libgithub "github.com/google/go-github/v74/github" slogctx "github.com/veqryn/slog-context" ) diff --git a/tools/pipeline/internal/pkg/github/copy_pull_request.go b/tools/pipeline/internal/pkg/github/copy_pull_request.go index 5b7d0c1fa0..374f3d4af1 100644 --- a/tools/pipeline/internal/pkg/github/copy_pull_request.go +++ b/tools/pipeline/internal/pkg/github/copy_pull_request.go @@ -15,7 +15,7 @@ import ( "slices" "strings" - libgithub "github.com/google/go-github/v68/github" + libgithub "github.com/google/go-github/v74/github" libgit "github.com/hashicorp/vault/tools/pipeline/internal/pkg/git" "github.com/jedib0t/go-pretty/v6/table" slogctx "github.com/veqryn/slog-context" diff --git a/tools/pipeline/internal/pkg/github/copy_pull_request_test.go b/tools/pipeline/internal/pkg/github/copy_pull_request_test.go index 09cb892b25..c59e88ad51 100644 --- a/tools/pipeline/internal/pkg/github/copy_pull_request_test.go +++ b/tools/pipeline/internal/pkg/github/copy_pull_request_test.go @@ -6,7 +6,7 @@ package github import ( "testing" - libgithub "github.com/google/go-github/v68/github" + libgithub "github.com/google/go-github/v74/github" "github.com/stretchr/testify/require" ) diff --git a/tools/pipeline/internal/pkg/github/create_backport.go b/tools/pipeline/internal/pkg/github/create_backport.go index 4ef5d2cf71..3c4e071f5b 100644 --- a/tools/pipeline/internal/pkg/github/create_backport.go +++ b/tools/pipeline/internal/pkg/github/create_backport.go @@ -15,7 +15,7 @@ import ( "slices" "strings" - libgithub "github.com/google/go-github/v68/github" + libgithub "github.com/google/go-github/v74/github" "github.com/hashicorp/vault/tools/pipeline/internal/pkg/changed" libgit "github.com/hashicorp/vault/tools/pipeline/internal/pkg/git" "github.com/hashicorp/vault/tools/pipeline/internal/pkg/releases" diff --git a/tools/pipeline/internal/pkg/github/create_backport_test.go b/tools/pipeline/internal/pkg/github/create_backport_test.go index af6702c217..364a18304a 100644 --- a/tools/pipeline/internal/pkg/github/create_backport_test.go +++ b/tools/pipeline/internal/pkg/github/create_backport_test.go @@ -8,7 +8,7 @@ import ( "errors" "testing" - libgithub "github.com/google/go-github/v68/github" + libgithub "github.com/google/go-github/v74/github" "github.com/hashicorp/vault/tools/pipeline/internal/pkg/changed" "github.com/hashicorp/vault/tools/pipeline/internal/pkg/releases" "github.com/stretchr/testify/require" diff --git a/tools/pipeline/internal/pkg/github/find_workflow_artifact.go b/tools/pipeline/internal/pkg/github/find_workflow_artifact.go new file mode 100644 index 0000000000..90e88dcf07 --- /dev/null +++ b/tools/pipeline/internal/pkg/github/find_workflow_artifact.go @@ -0,0 +1,200 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package github + +import ( + "cmp" + "context" + "encoding/json" + "errors" + "fmt" + "regexp" + "slices" + + gh "github.com/google/go-github/v74/github" + "github.com/jedib0t/go-pretty/v6/table" +) + +// FindWorkflowArtifactReq is a request to find an artifact associated with a +// workflow run. +type FindWorkflowArtifactReq struct { + ArtifactName string + ArtifactPattern string + Owner string + PullNumber int + Repo string + WorkflowName string + WriteToGithubOutput bool + compiledPattern *regexp.Regexp +} + +// FindWorkflowArtifactRes is a FindWorkflowArtifactReq response. +type FindWorkflowArtifactRes struct { + PR *gh.PullRequest `json:"pr,omitempty"` + Workflow *gh.Workflow `json:"workflow,omitempty"` + Run *WorkflowRun `json:"runs,omitempty"` + Artifact *gh.Artifact `json:"artifact,omitempty"` +} + +// Run performs the search to find an artifact associated with a workflow. +func (r *FindWorkflowArtifactReq) Run(ctx context.Context, client *gh.Client) (*FindWorkflowArtifactRes, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + var err error + res := &FindWorkflowArtifactRes{} + + // Validate our request. This also ensures that any pattern we've been given + // is a valid regex. + if err = r.validate(); err != nil { + return nil, fmt.Errorf("validating request: %w", err) + } + + // Get the workflow details for the repo + res.Workflow, err = getWorkflow(ctx, client, r.Owner, r.Repo, r.WorkflowName) + if err != nil { + return nil, fmt.Errorf("getting workflow: %w", err) + } + + // Get the pull request we're searching + res.PR, err = getPullRequest(ctx, client, r.Owner, r.Repo, r.PullNumber) + if err != nil { + return nil, fmt.Errorf("getting pull request: %w", err) + } + + // Get the workflow runs associated with the workflow and the PR + opts := &gh.ListWorkflowRunsOptions{ + Branch: res.PR.GetHead().GetRef(), + ExcludePullRequests: false, + HeadSHA: res.PR.GetHead().GetSHA(), + ListOptions: gh.ListOptions{PerPage: PerPageMax}, + Status: "success", + } + runs, err := getWorkflowRuns(ctx, client, r.Owner, r.Repo, res.Workflow.GetID(), opts) + if err != nil { + return nil, fmt.Errorf("getting workflow runs: %w", err) + } + + if len(runs) < 1 { + return nil, fmt.Errorf("no matching workflow runs are associated with the pull request: %w", err) + } + + // In instances where we have more than one run we want to get the artifact + // from the most recent run if possible. Search our runs in reverse order to + // find the most recent artifact. + slices.SortFunc(runs, func(a, b *WorkflowRun) int { + return cmp.Compare(*b.Run.RunAttempt, *a.Run.RunAttempt) + }) + + var artifacts gh.ArtifactList + for _, run := range runs { + artifacts, err = getWorkflowRunArtifacts(ctx, client, r.Owner, r.Repo, *run.Run.ID) + if err != nil { + return nil, fmt.Errorf("getting artifacts for workflow run %d: %w", *run.Run.ID, err) + } + + for _, art := range artifacts.Artifacts { + // If we've been given a name locate it by that + if r.ArtifactName != "" { + if art.GetName() == r.ArtifactName { + res.Artifact = art + + return res, nil + } + } else { + // Find it by regex + if r.compiledPattern.MatchString(art.GetName()) { + res.Artifact = art + + return res, nil + } + } + } + } + + return nil, errors.New("unable to find artifact matching given criteria") +} + +// validate ensures that we've been given the request configuration to perform +// the request. +func (r *FindWorkflowArtifactReq) validate() error { + if r == nil { + return errors.New("failed to initialize request") + } + + if r.Owner == "" { + return errors.New("no github organization has been provided") + } + + if r.Repo == "" { + return errors.New("no github repository has been provided") + } + + if r.PullNumber == 0 { + return errors.New("no github pull request number has been provided") + } + + if r.WorkflowName == "" { + return errors.New("no workflow name has been provided") + } + + if r.ArtifactName == "" && r.ArtifactPattern == "" { + return errors.New("no artifact name or pattern has been provided") + } + + if r.ArtifactName != "" && r.ArtifactPattern != "" { + return errors.New("you must provide only an artifact name or pattern") + } + + if r.ArtifactPattern != "" { + var err error + r.compiledPattern, err = regexp.Compile(r.ArtifactPattern) + if err != nil { + return fmt.Errorf("invalid artifact pattern: %w", err) + } + } + + return nil +} + +// ToJSON marshals the response to JSON. +func (r *FindWorkflowArtifactRes) ToJSON() ([]byte, error) { + b, err := json.Marshal(r) + if err != nil { + return nil, fmt.Errorf("marshaling find workflow artifact to JSON: %w", err) + } + + return b, nil +} + +// ToGithubOutput marshals just the artifact response to JSON. +func (r *FindWorkflowArtifactRes) ToGithubOutput() ([]byte, error) { + b, err := json.Marshal(r.Artifact) + if err != nil { + return nil, fmt.Errorf("marshaling find workflow artifact to GITHUB_OUTPUT JSON: %w", err) + } + + return b, nil +} + +// ToTable marshals the response to a text table. +func (r *FindWorkflowArtifactRes) ToTable() string { + t := table.NewWriter() + t.Style().Options.DrawBorder = false + t.Style().Options.SeparateColumns = false + t.Style().Options.SeparateFooter = false + t.Style().Options.SeparateHeader = false + t.Style().Options.SeparateRows = false + t.AppendHeader(table.Row{"name", "run id", "artifact id", "url"}) + t.AppendRow(table.Row{ + r.Artifact.GetName(), + r.Artifact.GetWorkflowRun().GetID(), + r.Artifact.GetID(), + r.Artifact.GetArchiveDownloadURL(), + }) + return t.Render() +} diff --git a/tools/pipeline/internal/pkg/github/list_changed_files.go b/tools/pipeline/internal/pkg/github/list_changed_files.go index bc5dc6a2c3..6e4fffb877 100644 --- a/tools/pipeline/internal/pkg/github/list_changed_files.go +++ b/tools/pipeline/internal/pkg/github/list_changed_files.go @@ -10,7 +10,7 @@ import ( "fmt" "strings" - gh "github.com/google/go-github/v68/github" + gh "github.com/google/go-github/v74/github" "github.com/hashicorp/vault/tools/pipeline/internal/pkg/changed" "github.com/jedib0t/go-pretty/v6/table" ) diff --git a/tools/pipeline/internal/pkg/github/list_workflow_runs.go b/tools/pipeline/internal/pkg/github/list_workflow_runs.go index 9ceff3a852..49128dd10e 100644 --- a/tools/pipeline/internal/pkg/github/list_workflow_runs.go +++ b/tools/pipeline/internal/pkg/github/list_workflow_runs.go @@ -10,7 +10,7 @@ import ( "net/http" "sync" - gh "github.com/google/go-github/v68/github" + gh "github.com/google/go-github/v74/github" ) // PerPageMax is the maximum number of entities to request for enpoints that @@ -75,7 +75,7 @@ func (r *ListWorkflowRunsReq) Run(ctx context.Context, client *gh.Client) (*List return nil, fmt.Errorf("validating request: %w", err) } - res.Workflow, err = r.getWorkflow(ctx, client) + res.Workflow, err = getWorkflow(ctx, client, r.Owner, r.Repo, r.WorkflowName) if err != nil { return nil, fmt.Errorf("getting workflow: %w", err) } @@ -141,32 +141,8 @@ func (r *ListWorkflowRunsReq) validate() error { return nil } -// getWorkflow attempts to locate the workflow associated with our workflow name. -func (r *ListWorkflowRunsReq) getWorkflow(ctx context.Context, client *gh.Client) (*gh.Workflow, error) { - opts := &gh.ListOptions{PerPage: PerPageMax} - for { - wfs, res, err := client.Actions.ListWorkflows(ctx, r.Owner, r.Repo, opts) - if err != nil { - return nil, err - } - - for _, wf := range wfs.Workflows { - if wf.GetName() == r.WorkflowName { - return wf, nil - } - } - - if res.NextPage == 0 { - return nil, fmt.Errorf("no workflow matching %s could be found", r.WorkflowName) - } - - opts.Page = res.NextPage - } -} - // getWorkflowRuns gets teh workflow runs associated with a workflow ID. func (r *ListWorkflowRunsReq) getWorkflowRuns(ctx context.Context, client *gh.Client, id int64) ([]*WorkflowRun, error) { - var runs []*WorkflowRun opts := &gh.ListWorkflowRunsOptions{ Actor: r.Actor, Branch: r.Branch, @@ -182,22 +158,7 @@ func (r *ListWorkflowRunsReq) getWorkflowRuns(ctx context.Context, client *gh.Cl opts.CheckSuiteID = r.CheckSuiteID } - for { - wfrs, res, err := client.Actions.ListWorkflowRunsByID(ctx, r.Owner, r.Repo, id, opts) - if err != nil { - return nil, err - } - - for _, r := range wfrs.WorkflowRuns { - runs = append(runs, &WorkflowRun{Run: r}) - } - - if res.NextPage == 0 { - return runs, nil - } - - opts.ListOptions.Page = res.NextPage - } + return getWorkflowRuns(ctx, client, r.Owner, r.Repo, id, opts) } // getWorkflowCheckRuns gets the check suite runs associated with the workflow runs. diff --git a/tools/pipeline/internal/pkg/github/pull_request.go b/tools/pipeline/internal/pkg/github/pull_request.go index c2c2ecd68d..d5d4c302b8 100644 --- a/tools/pipeline/internal/pkg/github/pull_request.go +++ b/tools/pipeline/internal/pkg/github/pull_request.go @@ -8,7 +8,7 @@ import ( "fmt" "log/slog" - libgithub "github.com/google/go-github/v68/github" + libgithub "github.com/google/go-github/v74/github" slogctx "github.com/veqryn/slog-context" ) diff --git a/tools/pipeline/internal/pkg/github/sync_branch_request.go b/tools/pipeline/internal/pkg/github/sync_branch_request.go index eef6c398c4..256f98499a 100644 --- a/tools/pipeline/internal/pkg/github/sync_branch_request.go +++ b/tools/pipeline/internal/pkg/github/sync_branch_request.go @@ -12,7 +12,7 @@ import ( "os" "path/filepath" - libgithub "github.com/google/go-github/v68/github" + libgithub "github.com/google/go-github/v74/github" libgit "github.com/hashicorp/vault/tools/pipeline/internal/pkg/git" "github.com/jedib0t/go-pretty/v6/table" slogctx "github.com/veqryn/slog-context" diff --git a/tools/pipeline/internal/pkg/github/templates_test.go b/tools/pipeline/internal/pkg/github/templates_test.go index d1dcbaf3e7..d614d0a3a0 100644 --- a/tools/pipeline/internal/pkg/github/templates_test.go +++ b/tools/pipeline/internal/pkg/github/templates_test.go @@ -8,7 +8,7 @@ import ( "io" "testing" - libgithub "github.com/google/go-github/v68/github" + libgithub "github.com/google/go-github/v74/github" "github.com/stretchr/testify/require" ) diff --git a/tools/pipeline/internal/pkg/github/workflows.go b/tools/pipeline/internal/pkg/github/workflows.go new file mode 100644 index 0000000000..72a4267656 --- /dev/null +++ b/tools/pipeline/internal/pkg/github/workflows.go @@ -0,0 +1,119 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package github + +import ( + "context" + "fmt" + "log/slog" + + gh "github.com/google/go-github/v74/github" + slogctx "github.com/veqryn/slog-context" +) + +// getWorkflow attempts to locate the workflow associated with our workflow name. +func getWorkflow( + ctx context.Context, + client *gh.Client, + owner string, + repo string, + name string, +) (*gh.Workflow, error) { + slog.Default().DebugContext(slogctx.Append(ctx, + slog.String("owner", owner), + slog.String("repo", repo), + slog.String("name", name), + ), "getting github actions workflow") + + opts := &gh.ListOptions{PerPage: PerPageMax} + for { + wfs, res, err := client.Actions.ListWorkflows(ctx, owner, repo, opts) + if err != nil { + return nil, err + } + + for _, wf := range wfs.Workflows { + if wf.GetName() == name { + return wf, nil + } + } + + if res.NextPage == 0 { + return nil, fmt.Errorf("no workflow matching %s could be found", name) + } + + opts.Page = res.NextPage + } +} + +// getWorkflowRuns gets the workflow runs associated with a workflow ID. +func getWorkflowRuns( + ctx context.Context, + client *gh.Client, + owner string, + repo string, + id int64, + opts *gh.ListWorkflowRunsOptions, +) ([]*WorkflowRun, error) { + slog.Default().DebugContext(slogctx.Append(ctx, + slog.String("owner", owner), + slog.String("repo", repo), + slog.Int64("id", id), + ), "getting github actions workflow runs") + + var runs []*WorkflowRun + opts.ListOptions = gh.ListOptions{PerPage: PerPageMax} + + for { + wfrs, res, err := client.Actions.ListWorkflowRunsByID(ctx, owner, repo, id, opts) + if err != nil { + return nil, err + } + + for _, r := range wfrs.WorkflowRuns { + runs = append(runs, &WorkflowRun{Run: r}) + } + + if res.NextPage == 0 { + return runs, nil + } + + opts.ListOptions.Page = res.NextPage + } +} + +// getWorkflowRunArtifacts gets the artifacts associated with a workflow run +func getWorkflowRunArtifacts( + ctx context.Context, + client *gh.Client, + owner string, + repo string, + id int64, +) (gh.ArtifactList, error) { + slog.Default().DebugContext(slogctx.Append(ctx, + slog.String("owner", owner), + slog.String("repo", repo), + slog.Int64("id", id), + ), "getting github actions workflow run artifacts") + + opts := &gh.ListOptions{PerPage: PerPageMax} + artifacts := gh.ArtifactList{} + + for { + arts, res, err := client.Actions.ListWorkflowRunArtifacts(ctx, owner, repo, id, opts) + if err != nil { + return artifacts, err + } + + newTotal := artifacts.GetTotalCount() + arts.GetTotalCount() + artifacts.TotalCount = &newTotal + artifacts.Artifacts = append(artifacts.Artifacts, arts.Artifacts...) + + if res.NextPage == 0 { + return artifacts, nil + } + + opts.Page = res.NextPage + } +} diff --git a/tools/pipeline/internal/pkg/hcp/client.go b/tools/pipeline/internal/pkg/hcp/client.go new file mode 100644 index 0000000000..07b2fbc5c2 --- /dev/null +++ b/tools/pipeline/internal/pkg/hcp/client.go @@ -0,0 +1,140 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package hcp + +import ( + "context" + "log/slog" + "net/http" + "os" + "sync" + + slogctx "github.com/veqryn/slog-context" +) + +// Client is the HCP client. +type Client struct { + Environment Environment + HTTPClient *http.Client + // Basic auth + Username string + Password string + once *sync.Once +} + +// ClientOpt is an option to NewClient. +type ClientOpt func(*Client) + +// Requester is an interface that defines a request that can be configured +// for an environment. +type Requester interface { + Request(Environment) (*http.Request, error) +} + +// Environment is an HCP portal environment +type Environment string + +const ( + EnvironmentUnknown Environment = "" + EnvironmentDev Environment = "dev" + EnvironmentInt Environment = "int" + EnvironmentProd Environment = "prod" +) + +// Addr is the URL for each environment +func (g Environment) Addr() string { + switch g { + case EnvironmentDev: + return "https://api.hcp.dev" + case EnvironmentInt: + return "https://api.hcp.to" + case EnvironmentProd: + return "https://api.hashicorp.cloud" + default: + return "" + } +} + +// NewClient takes none-or-more options and returns a new Client. +func NewClient(opts ...ClientOpt) *Client { + c := &Client{ + HTTPClient: http.DefaultClient, + } + + for _, opt := range opts { + opt(c) + } + + return c +} + +// WithEnvironment sets the client environment. +func WithEnvironment(env Environment) ClientOpt { + return func(c *Client) { + c.Environment = env + } +} + +// WithHTTPClient sets the client HTTP Client. +func WithHTTPClient(httpClient *http.Client) ClientOpt { + return func(c *Client) { + c.HTTPClient = httpClient + } +} + +// WithUsername sets the basic auth username for internal APIs. +func WithUsername(username string) ClientOpt { + return func(c *Client) { + c.Username = username + } +} + +// WithUsername sets the base auth password for internal APIs> +func WithPassword(password string) ClientOpt { + return func(c *Client) { + c.Password = password + } +} + +// WithLoadTokenFromEnv sets the basic auth username and token from known env +// vars. +func WithLoadAuthFromEnv() ClientOpt { + return func(client *Client) { + if username, ok := os.LookupEnv("HCP_USERNAME"); ok { + client.Username = username + } + if password, ok := os.LookupEnv("HCP_PASSWORD"); ok { + client.Password = password + } + } +} + +// Do takes in a Requester and performs the request. It returns the raw http +// Response. +func (c *Client) Do(ctx context.Context, req Requester) (*http.Response, error) { + logArgs := []any{ + slog.String("env", string(c.Environment)), + slog.String("api-addr", string(c.Environment.Addr())), + } + httpReq, err := req.Request(c.Environment) + if err != nil { + slog.Default().ErrorContext(slogctx.Append(ctx, + append(logArgs, slog.String("error", err.Error()))), + "performing request", + ) + return nil, err + } + + logArgs = append(logArgs, + slog.String("method", httpReq.Method), + slog.String("url", httpReq.URL.String()), + ) + + ctx = slogctx.Append(ctx, logArgs...) + slog.Default().DebugContext(ctx, "performing request") + httpReq.SetBasicAuth(c.Username, c.Password) + httpReq.WithContext(ctx) + + return c.HTTPClient.Do(httpReq) +} diff --git a/tools/pipeline/internal/pkg/hcp/get_latest_product_version.go b/tools/pipeline/internal/pkg/hcp/get_latest_product_version.go new file mode 100644 index 0000000000..fe4aade5da --- /dev/null +++ b/tools/pipeline/internal/pkg/hcp/get_latest_product_version.go @@ -0,0 +1,221 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package hcp + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "time" + + "github.com/jedib0t/go-pretty/v6/table" + slogctx "github.com/veqryn/slog-context" +) + +// GetLatestProductVersionReq is an HCP image service request to get the latest +// product version. It can also be used to get information for other images +// when configured with different constraints. +type GetLatestProductVersionReq struct { + ProductName string + ProductVersionConstraint string + HostManagerVersionConstraint string + CloudProvider string + CloudRegion string + Availability GetLatestProductVersionAvailability + ExcludeReleaseCandidates bool +} + +// GetLatestProductVersionAvailability describes the availability state of an +// image. +type GetLatestProductVersionAvailability string + +// GetLatestProductVersionRes is a response from a request to get the latest +// image from the HCP image service. +type GetLatestProductVersionRes struct { + Response *http.Response + Image *HCPImage `json:"image,omitempty"` +} + +// HCPRegion is a cloud region for the image. +type HCPRegion struct { + Provider string `json:"provider,omitempty"` + Region string `json:"region,omitempty"` +} + +// HCPImageReference is the image reference information. +type HCPImageReference struct { + ImageID string `json:"image_id,omitempty"` + Region *HCPRegion `json:"region,omitempty"` +} + +// HCPImage is an image in the HCP image service. +type HCPImage struct { + ID string `json:"id,omitempty"` + ProductName string `json:"product_name,omitempty"` + ProductVersion string `json:"product_version,omitempty"` + HostManagerVersion string `json:"host_manager_version,omitempty"` + Region *HCPRegion `json:"region,omitempty"` + Availability string `json:"availability,omitempty"` + AWS *HCPImageReference `json:"aws,omitempty"` + Azure *HCPImageReference `json:"azure,omitempty"` + OSVersion string `json:"os_version,omitempty"` + CreatedAt time.Time `json:"created_at,omitempty"` + UpdatedAt time.Time `json:"updated_at,omitempty"` +} + +const ( + GetLatestProductVersionAvailabilityUnknown GetLatestProductVersionAvailability = "" + GetLatestProductVersionAvailabilityDisabled GetLatestProductVersionAvailability = "disabled" + GetLatestProductVersionAvailabilityInternal GetLatestProductVersionAvailability = "internal" + GetLatestProductVersionAvailabilityPublic GetLatestProductVersionAvailability = "public" + GetLatestProductVersionAvailabilityBeta GetLatestProductVersionAvailability = "beta" +) + +// ID returns the availability into the corresponding integer enum. +func (g GetLatestProductVersionAvailability) ID() string { + switch g { + case GetLatestProductVersionAvailabilityDisabled: + return "1" + case GetLatestProductVersionAvailabilityInternal: + return "2" + case GetLatestProductVersionAvailabilityPublic: + return "3" + case GetLatestProductVersionAvailabilityBeta: + return "4" + default: + return "0" + } +} + +const imageServicePath = "image/2009-12-19/.internal/latestproductversion" + +// Request takes an environment and produces an HTTP request that the client +// can execute. +func (r *GetLatestProductVersionReq) Request(env Environment) (*http.Request, error) { + reqURL, err := url.JoinPath(env.Addr(), imageServicePath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodGet, reqURL, nil) + if err != nil { + return nil, err + } + + query := req.URL.Query() + if r.ProductName != "" { + query.Add("product_name", r.ProductName) + } + if r.ProductVersionConstraint != "" { + query.Add("product_version_constraint", r.ProductVersionConstraint) + } + if r.HostManagerVersionConstraint != "" { + query.Add("host_manager_version_constraint", r.HostManagerVersionConstraint) + } + if r.CloudProvider != "" { + query.Add("region.provider", r.CloudProvider) + } + if r.CloudRegion != "" { + query.Add("region.region", r.CloudRegion) + } + if r.Availability != "" { + query.Add("availability", r.Availability.ID()) + } + if r.ExcludeReleaseCandidates { + query.Add("exclude_release_candidates", fmt.Sprintf("%t", r.ExcludeReleaseCandidates)) + } + + req.URL.RawQuery = query.Encode() + + return req, nil +} + +// Run runs the request to find an HCP image that matches the request criteria. +func (r *GetLatestProductVersionReq) Run(ctx context.Context, client *Client) (*GetLatestProductVersionRes, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + res := &GetLatestProductVersionRes{} + + ctx = slogctx.Append(ctx, + slog.String("availability", string(r.Availability)), + slog.String("availability-id", r.Availability.ID()), + slog.String("cloud", r.CloudProvider), + slog.Bool("exclude-release-candidates", r.ExcludeReleaseCandidates), + slog.String("host-manager-version-constraint", r.HostManagerVersionConstraint), + slog.String("product", r.ProductName), + slog.String("product-version-constraint", r.ProductVersionConstraint), + slog.String("region", r.CloudRegion), + ) + slog.Default().DebugContext(ctx, "getting latest HCP product version") + + var err error + res.Response, err = client.Do(ctx, r) + if err != nil { + return res, err + } + + defer res.Response.Body.Close() + bytes, err := io.ReadAll(res.Response.Body) + if err != nil { + return res, err + } + + if err = json.Unmarshal(bytes, res); err != nil { + return res, err + } + + if res.Response.StatusCode > 299 { + return res, fmt.Errorf("received unexpected http response code: %d", res.Response.StatusCode) + } + + return res, nil +} + +// ToJSON marshals the response to JSON. +func (r *GetLatestProductVersionRes) ToJSON() ([]byte, error) { + b, err := json.Marshal(r) + if err != nil { + return nil, fmt.Errorf("marshaling latest HCP image response to JSON: %w", err) + } + + return b, nil +} + +// ToTable marshals the response to a text table. +func (r *GetLatestProductVersionRes) ToTable() table.Writer { + t := table.NewWriter() + t.Style().Options.DrawBorder = false + t.Style().Options.SeparateColumns = false + t.Style().Options.SeparateFooter = false + t.Style().Options.SeparateHeader = false + t.Style().Options.SeparateRows = false + t.AppendHeader(table.Row{"name", "id", "cloud", "region", "version", "created_at"}) + + var imageID string + if aws := r.Image.AWS; aws != nil { + imageID = aws.ImageID + } + if azure := r.Image.Azure; azure != nil { + imageID = azure.ImageID + } + + t.AppendRow(table.Row{ + r.Image.ProductName, + imageID, + r.Image.Region.Provider, + r.Image.Region.Region, + r.Image.ProductVersion, + r.Image.CreatedAt.String(), + }) + + return t +} diff --git a/tools/pipeline/internal/pkg/hcp/get_latest_product_version_test.go b/tools/pipeline/internal/pkg/hcp/get_latest_product_version_test.go new file mode 100644 index 0000000000..7e676a5add --- /dev/null +++ b/tools/pipeline/internal/pkg/hcp/get_latest_product_version_test.go @@ -0,0 +1,83 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package hcp + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func Test_GetLatestProductVersionReq_Request(t *testing.T) { + t.Parallel() + + for name, test := range map[string]struct { + req *GetLatestProductVersionReq + expectedURL string + }{ + "no query args": { + &GetLatestProductVersionReq{}, + "https://api.hcp.dev/image/2009-12-19/.internal/latestproductversion", + }, + "all query args": { + &GetLatestProductVersionReq{ + Availability: GetLatestProductVersionAvailabilityPublic, + CloudRegion: "us-east-1", + CloudProvider: "aws", + ExcludeReleaseCandidates: true, + ProductName: "vault", + ProductVersionConstraint: "1.21.0-beta1+ent-2cf0b2f", + }, + "https://api.hcp.dev/image/2009-12-19/.internal/latestproductversion?availability=3&exclude_release_candidates=true&product_name=vault&product_version_constraint=1.21.0-beta1%2Bent-2cf0b2f®ion.provider=aws®ion.region=us-east-1", + }, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + req, err := test.req.Request(EnvironmentDev) + require.NoError(t, err) + require.Equal(t, test.expectedURL, req.URL.String()) + }) + } +} + +const hcpImageJSON = `{"image":{"id":"c613a76a-7c7f-484d-b21f-f99603dacee7","product_name":"vault","product_version":"v1.21.0-beta1+ent-bf26069","host_manager_version":"0.2.1806001022+1f8c65e9","region":{"provider":"aws","region":"us-west-2"},"availability":"PUBLIC","aws":{"image_id":"ami-037fc9428b5fc8d6a","region":{"provider":"aws","region":"us-west-2"}},"os_version":"","created_at":"2025-07-23T06:45:04.029Z","updated_at":"2025-07-23T06:45:04.008Z"}}` + +func Test_GetLatestProductVersionRes_Unmarshal(t *testing.T) { + t.Parallel() + + createdAt, err := time.Parse(time.RFC3339, "2025-07-23T06:45:04.029Z") + require.NoError(t, err) + updatedAt, err := time.Parse(time.RFC3339, "2025-07-23T06:45:04.008Z") + require.NoError(t, err) + + res := &GetLatestProductVersionRes{} + require.NoError(t, json.Unmarshal([]byte(hcpImageJSON), res)) + expect := &GetLatestProductVersionRes{ + Image: &HCPImage{ + ID: "c613a76a-7c7f-484d-b21f-f99603dacee7", + ProductName: "vault", + ProductVersion: "v1.21.0-beta1+ent-bf26069", + HostManagerVersion: "0.2.1806001022+1f8c65e9", + Region: &HCPRegion{ + Provider: "aws", + Region: "us-west-2", + }, + Availability: "PUBLIC", + AWS: &HCPImageReference{ + ImageID: "ami-037fc9428b5fc8d6a", + Region: &HCPRegion{ + Provider: "aws", + Region: "us-west-2", + }, + }, + OSVersion: "", + CreatedAt: createdAt, + UpdatedAt: updatedAt, + }, + } + require.EqualValues(t, expect, res) +} diff --git a/tools/pipeline/internal/pkg/hcp/wait_for_image.go b/tools/pipeline/internal/pkg/hcp/wait_for_image.go new file mode 100644 index 0000000000..a9f4c20abf --- /dev/null +++ b/tools/pipeline/internal/pkg/hcp/wait_for_image.go @@ -0,0 +1,73 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package hcp + +import ( + "context" + "log/slog" + "time" + + "github.com/avast/retry-go/v4" + slogctx "github.com/veqryn/slog-context" +) + +// WaitForImageReq is a request to wait for an image to be available in the +// image service. +type WaitForImageReq struct { + Req *GetLatestProductVersionReq `json:"req,omitempty"` + Delay time.Duration `json:"delay,omitempty"` +} + +// WaitForImageRes is a response to a WaitForImageReq. +type WaitForImageRes struct { + Res *GetLatestProductVersionRes `json:"res,omitempty"` +} + +// Run runs the wait for image request. +func (r *WaitForImageReq) Run(ctx context.Context, client *Client) (*WaitForImageRes, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + slog.Default().DebugContext(ctx, "waiting for HCP image to be available") + + res := &WaitForImageRes{ + Res: &GetLatestProductVersionRes{}, + } + attempt := 0 + err := retry.Do( + func() error { + attempt++ + // Limit each request to the image service to 5 seconds max + reqCtx, reqCancel := context.WithTimeout(ctx, 5*time.Second) + defer reqCancel() + reqRes, err := r.Req.Run(reqCtx, client) + if reqRes != nil { + if reqRes.Response != nil { + res.Res.Response = reqRes.Response + } + if reqRes.Image != nil { + res.Res.Image = reqRes.Image + } + } + + if err != nil { + slog.Default().DebugContext( + slogctx.Append(ctx, + slog.Int("attempt", attempt), + slog.String("error", err.Error())), + "attempt to get HCP image details failed", + ) + } + + return err + }, + retry.Context(ctx), + retry.Delay(r.Delay), + ) + + return res, err +}