diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml new file mode 100644 index 0000000000..ee24bb9ab2 --- /dev/null +++ b/.github/workflows/build-docs.yml @@ -0,0 +1,20 @@ +--- +name: build-docs + +on: + push: + branches: + - docs/** + - backport/docs/** + +jobs: + # This is a workaround to allow docs PRs to fullfil the required branch + # protection rules which require that the 'build' workflow has + # succeeded before a PR is allowed to be merged. We validate that via the + # 'build-ensure-success' workflow that is triggered either by either the + # 'build' or 'build-docs' workflow. Without this trigger the required branch + # protection rules would never be met on docs branches. + trigger-build-success-workflow: + runs-on: ubuntu-latest + steps: + - run: exit 0 diff --git a/.github/workflows/build-ensure-success.yml b/.github/workflows/build-ensure-success.yml new file mode 100644 index 0000000000..aa0c8e8dce --- /dev/null +++ b/.github/workflows/build-ensure-success.yml @@ -0,0 +1,32 @@ +--- +name: ensure_build_workflow_succeeded + +on: + # Github's branch protection rules for required workflow checks are a bit + # cumbersome when you have many jobs. The required checks have to be configured + # at the workflow job level. As child workflows of build.yml and Enos can be + # dynamic and change often, there's no way to create or maintain a valid list + # of workflow checks that need to succeeded in order to merge. + # + # To work around this problem we've chosen to trigger this workflow on + # completion of all the jobs that are triggered by build or build-docs. This + # workflow inspects the conclusion of the build workflow event and either passes + # or fails. This allows our branch protection rules for the build workflow to + # depend only on this workflow succeeding, which can only happen if all child + # workflows of the build job have succeeded. + workflow_run: + workflows: [build, build-docs] + types: [completed] + +jobs: + ensure-all-build-workflows-succeeded: + name: Ensure that all workflows spawned by the build workflow succeeded + runs-on: ubuntu-latest + steps: + - if: ${{ github.event.workflow_run.conclusion != 'success' }} + uses: actions/github-script@v6 + with: + script: | + core.setFailed('One or more workflows spawned by the build job did not succeed. All build job workflows are required to pass before merge') + - if: ${{ github.event.workflow_run.conclusion == 'success' }} + run: echo "All build and integration workflows have succeeded!" diff --git a/.github/workflows/build-vault-oss.yml b/.github/workflows/build-vault-oss.yml new file mode 100644 index 0000000000..9dbc363367 --- /dev/null +++ b/.github/workflows/build-vault-oss.yml @@ -0,0 +1,109 @@ +--- +name: build_vault + +# This workflow is intended to be called by the build workflow for each Vault +# binary that needs to be built and packaged. The crt make targets that are +# utilized automatically determine build metadata and handle building and +# packing vault. + +on: + workflow_call: + inputs: + bundle-path: + required: false + type: string + cgo-enabled: + type: string + default: 0 + create-packages: + type: boolean + default: true + goos: + required: true + type: string + goarch: + required: true + type: string + go-tags: + type: string + go-version: + type: string + package-name: + type: string + default: vault + vault-version: + type: string + required: true + +jobs: + build: + runs-on: "ubuntu-latest" + name: Vault ${{ inputs.goos }} ${{ inputs.goarch }} v${{ inputs.vault-version }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: ${{ inputs.go-version }} + - name: Set up node and yarn + uses: actions/setup-node@v3 + with: + node-version: 14 + cache: yarn + cache-dependency-path: ui/yarn.lock + - name: Build UI + run: make crt-build-ui + - name: Build Vault + env: + CGO_ENABLED: ${{ inputs.cgo-enabled }} + GOARCH: ${{ inputs.goarch }} + GOOS: ${{ inputs.goos }} + GO_TAGS: ${{ inputs.go-tags }} + run: make crt-build + - name: Determine artifact basename + env: + GOARCH: ${{ inputs.goarch }} + GOOS: ${{ inputs.goos }} + run: echo "ARTIFACT_BASENAME=$(make crt-get-artifact-basename)" >> $GITHUB_ENV + - name: Bundle Vault + env: + BUNDLE_PATH: out/${{ env.ARTIFACT_BASENAME }}.zip + run: make crt-bundle + - uses: actions/upload-artifact@v3 + with: + name: ${{ env.ARTIFACT_BASENAME }}.zip + path: out/${{ env.ARTIFACT_BASENAME }}.zip + if-no-files-found: error + - if: ${{ inputs.create-packages }} + uses: hashicorp/actions-packaging-linux@v1 + with: + name: ${{ github.event.repository.name }} + description: Vault is a tool for secrets management, encryption as a service, and privileged access management. + arch: ${{ inputs.goarch }} + version: ${{ inputs.vault-version }} + maintainer: HashiCorp + homepage: https://github.com/hashicorp/vault + license: MPL-2.0 + binary: dist/${{ inputs.package-name }} + deb_depends: openssl + rpm_depends: openssl + config_dir: .release/linux/package/ + preinstall: .release/linux/preinst + postinstall: .release/linux/postinst + postremove: .release/linux/postrm + - if: ${{ inputs.create-packages }} + name: Determine package file names + run: | + echo "RPM_PACKAGE=$(basename out/*.rpm)" >> $GITHUB_ENV + echo "DEB_PACKAGE=$(basename out/*.deb)" >> $GITHUB_ENV + - if: ${{ inputs.create-packages }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.RPM_PACKAGE }} + path: out/${{ env.RPM_PACKAGE }} + if-no-files-found: error + - if: ${{ inputs.create-packages }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.DEB_PACKAGE }} + path: out/${{ env.DEB_PACKAGE }} + if-no-files-found: error diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0cf2cbb03a..5d24f2884d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,69 +2,63 @@ name: build on: workflow_dispatch: - push: + pull_request: + types: + - opened + - reopened + - synchronize branches-ignore: - docs/** - backport/docs/** + push: + branches: + - main + - release/** env: - PKG_NAME: "vault" - GO_TAGS: "ui" + PKG_NAME: vault jobs: - product-metadata: runs-on: ubuntu-latest outputs: - product-revision: ${{ steps.get-product-revision.outputs.product-revision }} - product-version: ${{ steps.get-product-version.outputs.product-version }} - product-base-version: ${{ steps.get-product-version.outputs.product-base-version }} - build-date: ${{ steps.get-build-date.outputs.build-date }} + build-date: ${{ steps.get-metadata.outputs.build-date }} + filepath: ${{ steps.generate-metadata-file.outputs.filepath }} + go-version: ${{ steps.get-metadata.outputs.go-version }} + package-name: ${{ steps.get-metadata.outputs.package-name }} + vault-revision: ${{ steps.get-metadata.outputs.vault-revision }} + vault-version: ${{ steps.get-metadata.outputs.vault-version }} + vault-base-version: ${{ steps.get-metadata.outputs.vault-base-version }} steps: - uses: actions/checkout@v3 - uses: actions/setup-go@v3 - - name: Get product revision - id: get-product-revision - run: echo "::set-output name=product-revision::$(git rev-parse HEAD)" - - name: Get product version - id: get-product-version + - name: Get metadata + id: get-metadata run: | - make version - IFS="-" read BASE_VERSION _other <<< "$(make version)" - echo "::set-output name=product-version::$(make version)" - echo "::set-output name=product-base-version::${BASE_VERSION}" - - name: Get build date - id: get-build-date - run: | - make build-date - echo "::set-output name=build-date::$(make build-date)" - - generate-metadata-file: - needs: product-metadata - runs-on: ubuntu-latest - outputs: - filepath: ${{ steps.generate-metadata-file.outputs.filepath }} - steps: - - name: 'Checkout directory' - uses: actions/checkout@v2 - - name: Generate metadata file + echo "build-date=$(make crt-get-date)" >> $GITHUB_OUTPUT + echo "package-name=${{ env.PKG_NAME }}" >> $GITHUB_OUTPUT + echo "go-version=$(cat ./.go-version)" >> $GITHUB_OUTPUT + echo "vault-base-version=$(make crt-get-version-base)" >> $GITHUB_OUTPUT + echo "vault-revision=$(make crt-get-revision)" >> $GITHUB_OUTPUT + echo "vault-version=$(make crt-get-version)" >> $GITHUB_OUTPUT + - uses: hashicorp/actions-generate-metadata@v1 id: generate-metadata-file - uses: hashicorp/actions-generate-metadata@v1 with: - version: ${{ needs.product-metadata.outputs.product-version }} - product: ${{ env.PKG_NAME }} - - uses: actions/upload-artifact@v2 + version: ${{ steps.get-metadata.outputs.vault-version }} + product: ${{ steps.get-metadata.outputs.package-name }} + - uses: actions/upload-artifact@v3 with: name: metadata.json path: ${{ steps.generate-metadata-file.outputs.filepath }} + if-no-files-found: error build-other: - needs: [ product-metadata ] - runs-on: ubuntu-latest + name: Build Vault Other + needs: product-metadata strategy: matrix: - goos: [ freebsd, windows, netbsd, openbsd, solaris ] - goarch: [ "386", "amd64", "arm" ] + goos: [freebsd, windows, netbsd, openbsd, solaris] + goarch: [386, amd64, arm] exclude: - goos: solaris goarch: 386 @@ -73,229 +67,99 @@ jobs: - goos: windows goarch: arm fail-fast: true - - name: Go ${{ matrix.goos }} ${{ matrix.goarch }} build - - steps: - - uses: actions/checkout@v2 - - name: Setup go - uses: actions/setup-go@v3 - with: - go-version-file: go.mod - - name: Setup node and yarn - uses: actions/setup-node@v2 - with: - node-version: '14' - cache: 'yarn' - cache-dependency-path: 'ui/yarn.lock' - - name: UI Build - run: | - cd ui - yarn install --ignore-optional - npm rebuild node-sass - yarn --verbose run build - cd .. - - name: Build - env: - GOOS: ${{ matrix.goos }} - GOARCH: ${{ matrix.goarch }} - CGO_ENABLED: 0 - run: | - mkdir dist out - GO_TAGS="${{ env.GO_TAGS }}" VAULT_VERSION=${{ needs.product-metadata.outputs.product-base-version }} VAULT_REVISION="$(git rev-parse HEAD)" VAULT_BUILD_DATE="${{ needs.product-metadata.outputs.build-date }}" make build - zip -r -j out/${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip dist/ - - uses: actions/upload-artifact@v2 - with: - name: ${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip - path: out/${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip + uses: ./.github/workflows/build-vault-oss.yml + with: + create-packages: false + goarch: ${{ matrix.goarch }} + goos: ${{ matrix.goos }} + go-tags: ui + go-version: ${{ needs.product-metadata.outputs.go-version }} + package-name: ${{ needs.product-metadata.outputs.package-name }} + vault-version: ${{ needs.product-metadata.outputs.vault-version }} + secrets: inherit build-linux: - needs: [ product-metadata ] - runs-on: ubuntu-latest + name: Build Vault Linux + needs: product-metadata strategy: matrix: goos: [linux] - goarch: ["arm", "arm64", "386", "amd64"] + goarch: [arm, arm64, 386, amd64] fail-fast: true - - name: Go ${{ matrix.goos }} ${{ matrix.goarch }} build - - steps: - - uses: actions/checkout@v2 - - - name: Setup go - uses: actions/setup-go@v3 - with: - go-version-file: go.mod - - name: Setup node and yarn - uses: actions/setup-node@v2 - with: - node-version: '14' - cache: 'yarn' - cache-dependency-path: 'ui/yarn.lock' - - name: UI Build - run: | - cd ui - yarn install --ignore-optional - npm rebuild node-sass - yarn --verbose run build - cd .. - - name: Build - env: - GOOS: ${{ matrix.goos }} - GOARCH: ${{ matrix.goarch }} - CGO_ENABLED: 0 - run: | - mkdir dist out - GO_TAGS="${{ env.GO_TAGS }}" VAULT_VERSION=${{ needs.product-metadata.outputs.product-base-version }} VAULT_REVISION="$(git rev-parse HEAD)" VAULT_BUILD_DATE="${{ needs.product-metadata.outputs.build-date }}" make build - zip -r -j out/${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip dist/ - - uses: actions/upload-artifact@v2 - with: - name: ${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip - path: out/${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip - - - name: Package - uses: hashicorp/actions-packaging-linux@v1 - with: - name: ${{ github.event.repository.name }} - description: "Vault is a tool for secrets management, encryption as a service, and privileged access management." - arch: ${{ matrix.goarch }} - version: ${{ needs.product-metadata.outputs.product-version }} - maintainer: "HashiCorp" - homepage: "https://github.com/hashicorp/vault" - license: "MPL-2.0" - binary: "dist/${{ env.PKG_NAME }}" - deb_depends: "openssl" - rpm_depends: "openssl" - config_dir: ".release/linux/package/" - preinstall: ".release/linux/preinst" - postinstall: ".release/linux/postinst" - postremove: ".release/linux/postrm" - - name: Add Package names to env - run: | - echo "RPM_PACKAGE=$(basename out/*.rpm)" >> $GITHUB_ENV - echo "DEB_PACKAGE=$(basename out/*.deb)" >> $GITHUB_ENV - - uses: actions/upload-artifact@v2 - with: - name: ${{ env.RPM_PACKAGE }} - path: out/${{ env.RPM_PACKAGE }} - - - uses: actions/upload-artifact@v2 - with: - name: ${{ env.DEB_PACKAGE }} - path: out/${{ env.DEB_PACKAGE }} - - build-darwin: - needs: [ product-metadata ] - runs-on: macos-latest - strategy: - matrix: - goos: [ darwin ] - goarch: [ "amd64", "arm64" ] - fail-fast: true - name: Go ${{ matrix.goos }} ${{ matrix.goarch }} build - steps: - - uses: actions/checkout@v2 - - - name: Setup go - uses: actions/setup-go@v3 - with: - go-version-file: go.mod - - name: Setup node and yarn - uses: actions/setup-node@v2 - with: - node-version: '14' - cache: 'yarn' - cache-dependency-path: 'ui/yarn.lock' - - name: UI Build - run: | - cd ui - yarn install --ignore-optional - npm rebuild node-sass - yarn --verbose run build - cd .. - - name: Build - env: - GOOS: ${{ matrix.goos }} - GOARCH: ${{ matrix.goarch }} - GO_TAGS: "${{ env.GO_TAGS }} netcgo" - CGO_ENABLED: 0 - run: | - mkdir dist out - GO_TAGS="${{ env.GO_TAGS }}" VAULT_VERSION=${{ needs.product-metadata.outputs.product-base-version }} VAULT_REVISION="$(git rev-parse HEAD)" VAULT_BUILD_DATE="${{ needs.product-metadata.outputs.build-date }}" make build - zip -r -j out/${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip dist/ - - uses: actions/upload-artifact@v2 - with: - name: ${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip - path: out/${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip - - build-docker: - name: Docker ${{ matrix.arch }} build - needs: - - product-metadata - - build-linux - runs-on: ubuntu-latest - strategy: - matrix: - arch: ["arm", "arm64", "386", "amd64"] - env: - repo: ${{github.event.repository.name}} - version: ${{needs.product-metadata.outputs.product-version}} - steps: - - uses: actions/checkout@v2 - - name: Docker Build (Action) - uses: hashicorp/actions-docker-build@v1 - with: - version: ${{env.version}} - target: default - arch: ${{matrix.arch}} - zip_artifact_name: ${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_linux_${{ matrix.arch }}.zip - tags: | - docker.io/hashicorp/${{env.repo}}:${{env.version}} - public.ecr.aws/hashicorp/${{env.repo}}:${{env.version}} - - enos-test-docker: - name: Enos Docker - # Only run the Enos workflow against branches that are created from the - # hashicorp/vault repository. This has the effect of limiting execution of - # Enos scenarios to branches that originate from authors that have write - # access to hashicorp/vault repository. This is required as Github Actions - # will not populate the required secrets for branches created by outside - # contributors in order to protect the secrets integrity. - if: "! github.event.pull_request.head.repo.fork" - needs: - - product-metadata - - build-docker - uses: ./.github/workflows/enos-run-k8s.yml + uses: ./.github/workflows/build-vault-oss.yml with: - artifact-build-date: "${{needs.product-metadata.outputs.build-date}}" - artifact-name: "${{github.event.repository.name}}_default_linux_amd64_${{needs.product-metadata.outputs.product-version}}_${{needs.product-metadata.outputs.product-revision}}.docker.tar" - artifact-revision: "${{needs.product-metadata.outputs.product-revision}}" - artifact-version: "${{needs.product-metadata.outputs.product-version}}" + goarch: ${{ matrix.goarch }} + goos: ${{ matrix.goos }} + go-tags: ui + go-version: ${{ needs.product-metadata.outputs.go-version }} + package-name: ${{ needs.product-metadata.outputs.package-name }} + vault-version: ${{ needs.product-metadata.outputs.vault-version }} secrets: inherit - build-ubi: - name: Red Hat UBI ${{ matrix.arch }} build + build-darwin: + name: Build Vault Darwin + needs: product-metadata + strategy: + matrix: + goos: [darwin] + goarch: [amd64, arm64] + fail-fast: true + uses: ./.github/workflows/build-vault-oss.yml + with: + create-packages: false + goarch: ${{ matrix.goarch }} + goos: ${{ matrix.goos }} + go-tags: ui netcgo + go-version: ${{ needs.product-metadata.outputs.go-version }} + package-name: ${{ needs.product-metadata.outputs.package-name }} + vault-version: ${{ needs.product-metadata.outputs.vault-version }} + secrets: inherit + + build-docker: + name: Build Vault Docker needs: - product-metadata - build-linux runs-on: ubuntu-latest strategy: matrix: - arch: ["amd64"] + arch: [arm, arm64, 386, amd64] env: - repo: ${{github.event.repository.name}} - version: ${{needs.product-metadata.outputs.product-version}} + repo: ${{ github.event.repository.name }} + version: ${{ needs.product-metadata.outputs.vault-version }} + steps: + - uses: actions/checkout@v3 + - uses: hashicorp/actions-docker-build@v1 + with: + version: ${{ env.version }} + target: default + arch: ${{ matrix.arch }} + zip_artifact_name: ${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.vault-version }}_linux_${{ matrix.arch }}.zip + tags: | + docker.io/hashicorp/${{ env.repo }}:${{ env.version }} + public.ecr.aws/hashicorp/${{ env.repo }}:${{ env.version }} + + build-ubi: + name: Build Vault Red Hat UBI + needs: + - product-metadata + - build-linux + runs-on: ubuntu-latest + strategy: + matrix: + arch: [amd64] + env: + repo: ${{ github.event.repository.name }} + version: ${{ needs.product-metadata.outputs.vault-version }} steps: - uses: actions/checkout@v2 - - name: Docker Build (Action) - uses: hashicorp/actions-docker-build@v1 + - uses: hashicorp/actions-docker-build@v1 with: - version: ${{env.version}} + version: ${{ env.version }} target: ubi - arch: ${{matrix.arch}} - zip_artifact_name: ${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.product-version }}_linux_${{ matrix.arch }}.zip - redhat_tag: quay.io/redhat-isv-containers/5f89bb5e0b94cf64cfeb500a:${{env.version}}-ubi + arch: ${{ matrix.arch }} + zip_artifact_name: ${{ env.PKG_NAME }}_${{ needs.product-metadata.outputs.vault-version }}_linux_${{ matrix.arch }}.zip + redhat_tag: quay.io/redhat-isv-containers/5f89bb5e0b94cf64cfeb500a:${{ env.version }}-ubi enos: name: Enos @@ -311,9 +175,29 @@ jobs: - build-linux uses: ./.github/workflows/enos-run.yml with: - artifact-build-date: "${{needs.product-metadata.outputs.build-date}}" - artifact-name: "vault_${{ needs.product-metadata.outputs.product-version }}_linux_amd64.zip" - artifact-revision: "${{needs.product-metadata.outputs.product-revision}}" - artifact-source: "crt" - artifact-version: "${{needs.product-metadata.outputs.product-version}}" + artifact-build-date: ${{ needs.product-metadata.outputs.build-date }} + artifact-name: vault_${{ needs.product-metadata.outputs.vault-version }}_linux_amd64.zip + artifact-revision: ${{ needs.product-metadata.outputs.vault-revision }} + artifact-source: crt + artifact-version: ${{ needs.product-metadata.outputs.vault-version }} + secrets: inherit + + enos-docker-k8s: + name: Enos Docker K8s + # Only run the Enos workflow against branches that are created from the + # hashicorp/vault repository. This has the effect of limiting execution of + # Enos scenarios to branches that originate from authors that have write + # access to hashicorp/vault repository. This is required as Github Actions + # will not populate the required secrets for branches created by outside + # contributors in order to protect the secrets integrity. + if: "! github.event.pull_request.head.repo.fork" + needs: + - product-metadata + - build-docker + uses: ./.github/workflows/enos-run-k8s.yml + with: + artifact-build-date: ${{ needs.product-metadata.outputs.build-date }} + artifact-name: ${{ github.event.repository.name }}_default_linux_amd64_${{ needs.product-metadata.outputs.vault-version }}_${{ needs.product-metadata.outputs.vault-revision }}.docker.tar + artifact-revision: ${{ needs.product-metadata.outputs.vault-revision }} + artifact-version: ${{ needs.product-metadata.outputs.vault-version }} secrets: inherit diff --git a/.github/workflows/enos-fmt.yml b/.github/workflows/enos-fmt.yml index c10395c8ba..298b2dc185 100644 --- a/.github/workflows/enos-fmt.yml +++ b/.github/workflows/enos-fmt.yml @@ -22,6 +22,6 @@ jobs: - uses: hashicorp/action-setup-enos@v1 with: github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} - - name: "check formatting" + - name: check formatting working-directory: ./enos run: make check-fmt diff --git a/.github/workflows/enos-run.yml b/.github/workflows/enos-run.yml index 3fbc3192e7..d934b98b3e 100644 --- a/.github/workflows/enos-run.yml +++ b/.github/workflows/enos-run.yml @@ -43,7 +43,7 @@ jobs: id: enos-matrix run: | [[ ${{ env.ARTIFACT_NAME }} == *"ent"* ]] && scenarioFile=$(cat ./.github/enos-run-matrices/${{ env.ARTIFACT_SOURCE }}-ent.json |jq -c .) || scenarioFile=$(cat ./.github/enos-run-matrices/${{ env.ARTIFACT_SOURCE }}-oss.json |jq -c .) - echo "::set-output name=matrix::$scenarioFile" + echo "matrix=$scenarioFile" >> $GITHUB_OUTPUT # Run Integration tests on Enos scenario matrix enos: name: Integration diff --git a/.github/workflows/enos-verify-stable.yml b/.github/workflows/enos-verify-stable.yml index c68836c0f3..6c0bf19baa 100644 --- a/.github/workflows/enos-verify-stable.yml +++ b/.github/workflows/enos-verify-stable.yml @@ -12,8 +12,8 @@ jobs: if: ${{ startsWith(github.event.client_payload.payload.branch, 'release/') }} uses: ./.github/workflows/enos-run.yml with: - artifact-source: "artifactory" - artifact-name: "${{ github.event.client_payload.payload.product }}_${{ github.event.client_payload.payload.version }}_linux_amd64.zip" - artifact-revision: "${{ github.event.client_payload.payload.sha }}" - artifact-version: "${{ github.event.client_payload.payload.version }}" + artifact-source: artifactory + artifact-name: ${{ github.event.client_payload.payload.product }}_${{ github.event.client_payload.payload.version }}_linux_amd64.zip + artifact-revision: ${{ github.event.client_payload.payload.sha }} + artifact-version: ${{ github.event.client_payload.payload.version }} secrets: inherit diff --git a/Makefile b/Makefile index 0827093d03..b5f77760c2 100644 --- a/Makefile +++ b/Makefile @@ -254,18 +254,48 @@ ci-verify: .NOTPARALLEL: ember-dist ember-dist-dev -.PHONY: build -# This is used for release builds by .github/workflows/build.yml -build: - @echo "--> Building Vault $(VAULT_VERSION)" - @go build -v -tags "$(GO_TAGS)" -ldflags " -s -w -X github.com/hashicorp/vault/sdk/version.Version=$(VAULT_VERSION) -X github.com/hashicorp/vault/sdk/version.GitCommit=$(VAULT_REVISION) -X github.com/hashicorp/vault/sdk/version.BuildDate=$(VAULT_BUILD_DATE)" -o dist/ +# These crt targets are used for release builds by .github/workflows/build.yml +# and for artifact_source:local Enos scenario variants. +.PHONY: crt-build +crt-build: + @$(CURDIR)/scripts/crt-builder.sh build -.PHONY: version -# This is used for release builds by .github/workflows/build.yml -version: - @$(CURDIR)/scripts/version.sh sdk/version/version_base.go +.PHONY: crt-build-ui +crt-build-ui: + @$(CURDIR)/scripts/crt-builder.sh build-ui -.PHONY: build-date -# This is used for release builds by .github/workflows/build.yml -build-date: - @$(CURDIR)/scripts/build_date.sh +.PHONY: crt-bundle +crt-bundle: + @$(CURDIR)/scripts/crt-builder.sh bundle + +.PHONY: crt-get-artifact-basename +crt-get-artifact-basename: + @$(CURDIR)/scripts/crt-builder.sh artifact-basename + +.PHONY: crt-get-date +crt-get-date: + @$(CURDIR)/scripts/crt-builder.sh date + +.PHONY: crt-get-revision +crt-get-revision: + @$(CURDIR)/scripts/crt-builder.sh revision + +.PHONY: crt-get-version +crt-get-version: + @$(CURDIR)/scripts/crt-builder.sh version + +.PHONY: crt-get-version-base +crt-get-version-base: + @$(CURDIR)/scripts/crt-builder.sh version-base + +.PHONY: crt-get-version-pre +crt-get-version-pre: + @$(CURDIR)/scripts/crt-builder.sh version-pre + +.PHONY: crt-get-version-meta +crt-get-version-meta: + @$(CURDIR)/scripts/crt-builder.sh version-meta + +.PHONY: crt-prepare-legal +crt-prepare-legal: + @$(CURDIR)/scripts/crt-builder.sh prepare-legal diff --git a/enos/enos-scenario-agent.hcl b/enos/enos-scenario-agent.hcl index 81cd12c4a1..a785c61012 100644 --- a/enos/enos-scenario-agent.hcl +++ b/enos/enos-scenario-agent.hcl @@ -36,6 +36,7 @@ scenario "agent" { arm64 = "t4g.small" } vault_instance_type = coalesce(var.vault_instance_type, local.vault_instance_types[matrix.arch]) + vault_license_path = abspath(var.vault_license_path != null ? var.vault_license_path : joinpath(path.root, "./support/vault.hclic")) } step "get_local_metadata" { @@ -91,7 +92,7 @@ scenario "agent" { module = module.read_license variables { - file_name = abspath(joinpath(path.root, "./support/vault.hclic")) + file_name = local.vault_license_path } } diff --git a/enos/enos-scenario-autopilot.hcl b/enos/enos-scenario-autopilot.hcl index f2e9db6e7a..881613759b 100644 --- a/enos/enos-scenario-autopilot.hcl +++ b/enos/enos-scenario-autopilot.hcl @@ -36,13 +36,14 @@ scenario "autopilot" { arm64 = "t4g.small" } vault_instance_type = coalesce(var.vault_instance_type, local.vault_instance_types[matrix.arch]) + vault_license_path = abspath(var.vault_license_path != null ? var.vault_license_path : joinpath(path.root, "./support/vault.hclic")) } step "build_vault" { module = "build_${matrix.artifact_source}" variables { - build_tags = try(var.vault_local_build_tags, local.build_tags[matrix.edition]) + build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : local.build_tags[matrix.edition] bundle_path = local.bundle_path goarch = matrix.arch goos = "linux" @@ -85,7 +86,7 @@ scenario "autopilot" { module = module.read_license variables { - file_name = abspath(joinpath(path.root, "./support/vault.hclic")) + file_name = local.vault_license_path } } diff --git a/enos/enos-scenario-smoke.hcl b/enos/enos-scenario-smoke.hcl index 6091e66c49..56b53d006a 100644 --- a/enos/enos-scenario-smoke.hcl +++ b/enos/enos-scenario-smoke.hcl @@ -39,6 +39,7 @@ scenario "smoke" { arm64 = "t4g.small" } vault_instance_type = coalesce(var.vault_instance_type, local.vault_instance_types[matrix.arch]) + vault_license_path = abspath(var.vault_license_path != null ? var.vault_license_path : joinpath(path.root, "./support/vault.hclic")) } step "get_local_metadata" { @@ -50,7 +51,7 @@ scenario "smoke" { module = "build_${matrix.artifact_source}" variables { - build_tags = try(var.vault_local_build_tags, local.build_tags[matrix.edition]) + build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : local.build_tags[matrix.edition] bundle_path = local.bundle_path goarch = matrix.arch goos = "linux" @@ -94,7 +95,7 @@ scenario "smoke" { module = module.read_license variables { - file_name = abspath(joinpath(path.root, "./support/vault.hclic")) + file_name = local.vault_license_path } } diff --git a/enos/enos-scenario-upgrade.hcl b/enos/enos-scenario-upgrade.hcl index cc85077677..2baf11d2f4 100644 --- a/enos/enos-scenario-upgrade.hcl +++ b/enos/enos-scenario-upgrade.hcl @@ -39,13 +39,14 @@ scenario "upgrade" { arm64 = "t4g.small" } vault_instance_type = coalesce(var.vault_instance_type, local.vault_instance_types[matrix.arch]) + vault_license_path = abspath(var.vault_license_path != null ? var.vault_license_path : joinpath(path.root, "./support/vault.hclic")) } step "build_vault" { module = "build_${matrix.artifact_source}" variables { - build_tags = try(var.vault_local_build_tags, local.build_tags[matrix.edition]) + build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : local.build_tags[matrix.edition] bundle_path = local.bundle_path goarch = matrix.arch goos = "linux" @@ -89,7 +90,7 @@ scenario "upgrade" { module = module.read_license variables { - file_name = abspath(joinpath(path.root, "./support/vault.hclic")) + file_name = local.vault_license_path } } diff --git a/enos/modules/build_local/main.tf b/enos/modules/build_local/main.tf index 3dc189b0e8..f8c12b1a2a 100644 --- a/enos/modules/build_local/main.tf +++ b/enos/modules/build_local/main.tf @@ -58,10 +58,12 @@ variable "vault_product_version" { } resource "enos_local_exec" "build" { - content = templatefile("${path.module}/templates/build.sh", { - bundle_path = var.bundle_path, - build_tags = join(" ", var.build_tags) - goarch = var.goarch - goos = var.goos - }) + scripts = ["${path.module}/scripts/build.sh"] + + environment = { + BUNDLE_PATH = var.bundle_path, + GO_TAGS = join(" ", var.build_tags) + GOARCH = var.goarch + GOOS = var.goos + } } diff --git a/enos/modules/build_local/scripts/build.sh b/enos/modules/build_local/scripts/build.sh new file mode 100755 index 0000000000..3478079c20 --- /dev/null +++ b/enos/modules/build_local/scripts/build.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -eux -o pipefail + +# Install yarn so we can build the UI +npm install --global yarn || true + +export CGO_ENABLED=0 + +root_dir="$(git rev-parse --show-toplevel)" +pushd "$root_dir" > /dev/null +make crt-build-ui crt-build crt-bundle +popd > /dev/null diff --git a/enos/modules/build_local/templates/build.sh b/enos/modules/build_local/templates/build.sh deleted file mode 100755 index 661df529bf..0000000000 --- a/enos/modules/build_local/templates/build.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -eux -o pipefail - -# Requirements -npm install --global yarn || true - -# Set up the environment for building Vault. -root_dir="$(git rev-parse --show-toplevel)" - -pushd "$root_dir" > /dev/null - -export GO_TAGS=${build_tags} -export CGO_ENABLED=0 - -IFS="-" read -r BASE_VERSION _other <<< "$(make version)" -export VAULT_VERSION=$BASE_VERSION - -build_date="$(make build-date)" -export VAULT_BUILD_DATE=$build_date - -revision="$(git rev-parse HEAD)" -export VAULT_REVISION=$revision -popd > /dev/null - -# Go to the UI directory of the Vault repo and build the UI -pushd "$root_dir/ui" > /dev/null -yarn install --ignore-optional -npm rebuild node-sass -yarn --verbose run build -popd > /dev/null - -# Build for linux/amd64 and create a bundle since we're deploying it to linux/amd64 -pushd "$root_dir" > /dev/null -export GOARCH=${goarch} -export GOOS=${goos} -make build - -zip -r -j ${bundle_path} dist/ -popd > /dev/null diff --git a/enos/modules/get_local_metadata/scripts/build_date.sh b/enos/modules/get_local_metadata/scripts/build_date.sh index fbb7f9eb57..d0528554e7 100755 --- a/enos/modules/get_local_metadata/scripts/build_date.sh +++ b/enos/modules/get_local_metadata/scripts/build_date.sh @@ -1,10 +1,6 @@ #!/bin/env bash set -eu -o pipefail -# Set up the environment for building Vault. -root_dir="$(git rev-parse --show-toplevel)" - -pushd "$root_dir" > /dev/null - -IFS="-" read -r VAULT_BUILD_DATE _other <<< "$(make build-date)" -echo $VAULT_BUILD_DATE +pushd "$(git rev-parse --show-toplevel)" > /dev/null +make crt-get-date +popd > /dev/null diff --git a/enos/modules/get_local_metadata/scripts/version.sh b/enos/modules/get_local_metadata/scripts/version.sh index 13970ead97..ef0b91f378 100755 --- a/enos/modules/get_local_metadata/scripts/version.sh +++ b/enos/modules/get_local_metadata/scripts/version.sh @@ -1,10 +1,6 @@ #!/bin/env bash set -eu -o pipefail -# Set up the environment for building Vault. -root_dir="$(git rev-parse --show-toplevel)" - -pushd "$root_dir" > /dev/null - -IFS="-" read -r VAULT_VERSION _other <<< "$(make version)" -echo $VAULT_VERSION +pushd "$(git rev-parse --show-toplevel)" > /dev/null +make crt-get-version +popd > /dev/null diff --git a/scripts/build.sh b/scripts/build.sh index b2e1302cb4..c041e0574e 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -18,10 +18,10 @@ cd "$DIR" BUILD_TAGS="${BUILD_TAGS:-"vault"}" # Get the git commit -GIT_COMMIT="$(git rev-parse HEAD)" +GIT_COMMIT="$("$SOURCE_DIR"/crt-builder.sh revision)" GIT_DIRTY="$(test -n "`git status --porcelain`" && echo "+CHANGES" || true)" -BUILD_DATE=$("$SOURCE_DIR"/build_date.sh) +BUILD_DATE="$("$SOURCE_DIR"/crt-builder.sh date)" GOPATH=${GOPATH:-$(${GO_CMD} env GOPATH)} case $(uname) in diff --git a/scripts/build_date.sh b/scripts/build_date.sh deleted file mode 100755 index 56504595f8..0000000000 --- a/scripts/build_date.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -# it's tricky to do an RFC3339 format in a cross platform way, so we hardcode UTC -DATE_FORMAT="%Y-%m-%dT%H:%M:%SZ" - -# we're using this for build date because it's stable across platform builds -git show --no-show-signature -s --format=%cd --date=format:"$DATE_FORMAT" HEAD diff --git a/scripts/crt-builder.sh b/scripts/crt-builder.sh new file mode 100755 index 0000000000..f69797efcb --- /dev/null +++ b/scripts/crt-builder.sh @@ -0,0 +1,234 @@ +#!/usr/bin/env bash + +# The crt-builder is used to detemine build metadata and create Vault builds. +# We use it in build-vault.yml for building release artifacts with CRT. It is +# also used by Enos for artifact_source:local scenario variants. + +set -euo pipefail + +# We don't want to get stuck in some kind of interactive pager +export GIT_PAGER=cat + +# Get the full version information +function version() { + local version + local prerelease + local metadata + + version=$(version_base) + prerelease=$(version_pre) + metadata=$(version_metadata) + + if [ -n "$metadata" ] && [ -n "$prerelease" ]; then + echo "$version-$prerelease+$metadata" + elif [ -n "$metadata" ]; then + echo "$version+$metadata" + elif [ -n "$prerelease" ]; then + echo "$version-$prerelease" + else + echo "$version" + fi +} + +# Get the base version +function version_base() { + : "${VAULT_VERSION:=""}" + + if [ -n "$VAULT_VERSION" ]; then + echo "$VAULT_VERSION" + return + fi + + : "${VERSION_FILE:=$(repo_root)/sdk/version/version_base.go}" + awk '$1 == "Version" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < "$VERSION_FILE" +} + +# Get the version pre-release +function version_pre() { + : "${VAULT_PRERELEASE:=""}" + + if [ -n "$VAULT_PRERELEASE" ]; then + echo "$VAULT_PRERELEASE" + return + fi + + : "${VERSION_FILE:=$(repo_root)/sdk/version/version_base.go}" + awk '$1 == "VersionPrerelease" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < "$VERSION_FILE" +} + +# Get the version metadata, which is commonly the edition +function version_metadata() { + : "${VAULT_METADATA:=""}" + + if [ -n "$VAULT_METADATA" ]; then + echo "$VAULT_METADATA" + return + fi + + : "${VERSION_FILE:=$(repo_root)/sdk/version/version_base.go}" + awk '$1 == "VersionMetadata" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < "$VERSION_FILE" +} + +# Get the build date from the latest commit since it can be used across all +# builds +function build_date() { + # It's tricky to do an RFC3339 format in a cross platform way, so we hardcode UTC + : "${DATE_FORMAT:="%Y-%m-%dT%H:%M:%SZ"}" + git show --no-show-signature -s --format=%cd --date=format:"$DATE_FORMAT" HEAD +} + +# Get the revision, which is the latest commit SHA +function build_revision() { + git rev-parse HEAD +} + +# Determine our repository by looking at our origin URL +function repo() { + basename -s .git "$(git config --get remote.origin.url)" +} + +# Determine the root directory of the repository +function repo_root() { + git rev-parse --show-toplevel +} + +# Determine the artifact basename based on metadata +function artifact_basename() { + : "${PKG_NAME:="vault"}" + : "${GOOS:=$(go env GOOS)}" + : "${GOARCH:=$(go env GOARCH)}" + + echo "${PKG_NAME}_$(version)_${GOOS}_${GOARCH}" +} + +# Build the UI +function build_ui() { + local repo_root + repo_root=$(repo_root) + + pushd "$repo_root" + mkdir -p http/web_ui + popd + pushd "$repo_root/ui" + yarn install --ignore-optional + npm rebuild node-sass + yarn --verbose run build + popd +} + +# Build Vault +function build() { + local version + local revision + local prerelease + local build_date + local ldflags + local msg + + # Get or set our basic build metadata + version=$(version_base) + revision=$(build_revision) + metadata=$(version_metadata) + prerelease=$(version_pre) + build_date=$(build_date) + : "${GO_TAGS:=""}" + : "${KEEP_SYMBOLS:=""}" + + # Build our ldflags + msg="--> Building Vault v$version, revision $revision, built $build_date" + + # Strip the symbol and dwarf information by default + if [ -n "$KEEP_SYMBOLS" ]; then + ldflags="" + else + ldflags="-s -w " + fi + + ldflags="${ldflags}-X github.com/hashicorp/vault/sdk/version.Version=$version -X github.com/hashicorp/vault/sdk/version.GitCommit=$revision -X github.com/hashicorp/vault/sdk/version.BuildDate=$build_date" + + if [ -n "$prerelease" ]; then + msg="${msg}, prerelease ${prerelease}" + ldflags="${ldflags} -X github.com/hashicorp/vault/sdk/version.VersionPrerelease=$prerelease" + fi + + if [ -n "$metadata" ]; then + msg="${msg}, metadata ${VAULT_METADATA}" + ldflags="${ldflags} -X github.com/hashicorp/vault/sdk/version.VersionMetadata=$metadata" + fi + + # Build vault + echo "$msg" + pushd "$(repo_root)" + mkdir -p dist + mkdir -p out + set -x + go build -v -tags "$GO_TAGS" -ldflags "$ldflags" -o dist/ + set +x + popd +} + +# Bundle the dist directory +function bundle() { + : "${BUNDLE_PATH:=$(repo_root)/vault.zip}" + echo "--> Bundling dist/* to $BUNDLE_PATH" + zip -r -j "$BUNDLE_PATH" dist/ +} + +# Prepare legal requirements for packaging +function prepare_legal() { + : "${PKG_NAME:="vault"}" + + pushd "$(repo_root)" + mkdir -p dist + curl -o dist/EULA.txt https://eula.hashicorp.com/EULA.txt + curl -o dist/TermsOfEvaluation.txt https://eula.hashicorp.com/TermsOfEvaluation.txt + mkdir -p ".release/linux/package/usr/share/doc/$PKG_NAME" + cp dist/EULA.txt ".release/linux/package/usr/share/doc/$PKG_NAME/EULA.txt" + cp dist/TermsOfEvaluation.txt ".release/linux/package/usr/share/doc/$PKG_NAME/TermsOfEvaluation.txt" + popd +} + +# Run the CRT Builder +function main() { + case $1 in + artifact-basename) + artifact_basename + ;; + build) + build + ;; + build-ui) + build_ui + ;; + bundle) + bundle + ;; + date) + build_date + ;; + prepare-legal) + prepare_legal + ;; + revision) + build_revision + ;; + version) + version + ;; + version-base) + version_base + ;; + version-pre) + version_pre + ;; + version-meta) + version_metadata + ;; + *) + echo "unknown sub-command" >&2 + exit 1 + ;; + esac +} + +main "$@" diff --git a/scripts/version.sh b/scripts/version.sh deleted file mode 100755 index 41177a45fa..0000000000 --- a/scripts/version.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -version_file=$1 -version=$(awk '$1 == "Version" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < "${version_file}") -prerelease=$(awk '$1 == "VersionPrerelease" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < "${version_file}") - -if [ -n "$prerelease" ]; then - echo "${version}-${prerelease}" -else - echo "${version}" -fi