diff --git a/.build/entrypoint.sh b/.build/entrypoint.sh new file mode 100644 index 0000000000..a7a7dac306 --- /dev/null +++ b/.build/entrypoint.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +set -e + +fail() { + echo "$1" 1>&2 + exit 1 +} + +[[ -z "$GOARCH" ]] && fail "A GOARCH has not been defined" +[[ -z "$GITHUB_TOKEN" ]] && fail "A GITHUB_TOKEN has not been defined" + +host_arch="$(dpkg --print-architecture)" +host_arch="${host_arch##*-}" +if [[ "$host_arch" != "$GOARCH" ]]; then + # We're building for a different architecture than our target host OS so + # we have to tell the Go compiler to use the correct C cross-compiler for + # our target instead of relying on the host C compiler. + # + # https://packages.ubuntu.com/search?suite=noble§ion=all&arch=any&keywords=linux-gnu-gcc&searchon=contents + case "$GOARCH" in + amd64) + export CC=x86_64-linux-gnu-gcc + ;; + arm64) + export CC=aarch64-linux-gnu-gcc + ;; + s390x) + export CC=s390x-linux-gnu-gcc + ;; + *) + fail "Building for $GOARCH has not been implemented" + ;; + esac +fi + +# Assume that /build is where we've mounted the vault repo. +git config --global --add safe.directory /build +git config --global url."https://${GITHUB_TOKEN}@github.com".insteadOf "https://github.com" + +# Exec our command +cd build || exit 1 +exec "$@" diff --git a/.build/go.sh b/.build/go.sh new file mode 100644 index 0000000000..5790e46509 --- /dev/null +++ b/.build/go.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 +set -e + +host_arch="$(dpkg --print-architecture)" +host_arch="${host_arch##*-}" +curl -L "https://go.dev/dl/go${GO_VERSION}.linux-${host_arch}.tar.gz" | tar -C /opt -zxv diff --git a/.build/system.sh b/.build/system.sh new file mode 100644 index 0000000000..bcb2ff271a --- /dev/null +++ b/.build/system.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +set -e + +export DEBIAN_FRONTEND=noninteractive + +install() { + apt-get install -y "$@" +} + +# Install our cross building tools +# https://packages.ubuntu.com/search?suite=noble§ion=all&arch=any&keywords=crossbuild-essential&searchon=names + +apt-get update +apt-get install -y --no-install-recommends build-essential \ + gcc-s390x-linux-gnu \ + crossbuild-essential-s390x \ + ca-certificates \ + curl \ + git + +host_arch="$(dpkg --print-architecture)" +host_arch="${host_arch##*-}" +case "$host_arch" in + amd64) + install crossbuild-essential-arm64 gcc-aarch64-linux-gnu + ;; + arm64) + install gcc-x86-64-linux-gnu + ;; + *) + echo "Building on $host_arch has not been implemented" 1>&2 + exit 1 + ;; +esac + +# Clean up after ourselves for a minimal image +apt-get clean +rm -rf /var/lib/apt/lists/* diff --git a/.github/actions/build-vault/action.yml b/.github/actions/build-vault/action.yml index 28c884b0a0..af2057be37 100644 --- a/.github/actions/build-vault/action.yml +++ b/.github/actions/build-vault/action.yml @@ -11,59 +11,44 @@ description: | inputs: github-token: - type: string description: An elevated Github token to access private Go modules if necessary. default: "" cgo-enabled: - type: number description: Enable or disable CGO during the build. - default: 0 + default: "0" create-docker-container: - type: boolean description: Package the binary into a Docker/AWS container. - default: true + default: "true" create-redhat-container: - type: boolean description: Package the binary into a Redhat container. - default: false + default: "false" create-packages: - type: boolean description: Package the binaries into deb and rpm formats. - default: true + default: "true" goos: - type: string description: The Go GOOS value environment variable to set during the build. goarch: - type: string description: The Go GOARCH value environment variable to set during the build. goarm: - type: string description: The Go GOARM value environment variable to set during the build. default: "" goexperiment: - type: string description: Which Go experiments to enable. default: "" go-tags: - type: string description: A comma separated list of tags to pass to the Go compiler during build. default: "" package-name: - type: string description: The name to use for the linux packages. default: ${{ github.event.repository.name }} vault-binary-name: - type: string description: The name of the vault binary. default: vault vault-edition: - type: string description: The edition of vault to build. vault-version: - type: string description: The version metadata to inject into the build via the linker. web-ui-cache-key: - type: string description: The cache key for restoring the pre-built web UI artifact. outputs: @@ -74,29 +59,12 @@ outputs: runs: using: composite steps: - - name: Ensure zstd is available for actions/cache - # actions/cache restores based on cache key and "cache version", the former is unique to the - # build job or web UI, the latter is a hash which is based on the runner OS, the paths being - # cached, and the program used to compress it. Most of our workflows will use zstd to compress - # the cached artifact so we have to have it around for our machines to get both a version match - # and to decompress it. Most runners include zstd by default but there are exception like - # our Ubuntu 20.04 compatibility runners which do not. - shell: bash - run: which zstd || (sudo apt update && sudo apt install -y zstd) - - uses: ./.github/actions/set-up-go + - id: set-up-go + uses: ./.github/actions/set-up-go with: github-token: ${{ inputs.github-token }} - - uses: ./.github/actions/install-external-tools - - if: inputs.goarch == 's390x' && inputs.vault-edition == 'ent.hsm' - name: Configure CGO compiler for HSM edition on s390x - shell: bash - run: | - sudo apt-get update - sudo apt-get install -y gcc-multilib-s390x-linux-gnu - { - echo "CC=s390x-linux-gnu-gcc" - echo "CC_FOR_TARGET=s390x-linux-gnu-gcc" - } | tee -a "$GITHUB_ENV" + - if: inputs.cgo-enabled == '0' + uses: ./.github/actions/install-external-tools - if: inputs.vault-edition != 'ce' name: Configure Git shell: bash @@ -126,15 +94,27 @@ runs: build_step_name='Vault ${{ inputs.goos }} ${{ inputs.goarch }} v${{ inputs.vault-version }}+${{ inputs.vault-edition }}' package_version='${{ inputs.vault-version }}+ent' # this should always be +ent here regardless of enterprise edition fi + # Generate a builder cache key that considers anything that might change + # our build container, including: + # - The Go version we're building with + # - External Go build tooling as defined in tools/tools.sh + # - The Dockerfile or .build directory + # - The build-vault Github action + docker_sha=$(git ls-tree HEAD Dockerfile --object-only --abbrev=5) + build_sha=$(git ls-tree HEAD .build --object-only --abbrev=5) + tools_sha=$(git ls-tree HEAD tools/tools.sh --object-only --abbrev=5) + github_sha=$(git ls-tree HEAD .github/actions/build-vault --object-only --abbrev=5) { echo "artifact-basename=$(make ci-get-artifact-basename)" echo "binary-path=dist/${{ inputs.vault-binary-name }}" echo "build-step-name=${build_step_name}" + echo "vault-builder-cache-key=${docker_sha}-${build_sha}-${tools_sha}-${github_sha}-$(cat .go-version)" echo "package-version=${package_version}" } | tee -a "$GITHUB_OUTPUT" - - name: ${{ steps.metadata.outputs.build-step-name }} + - if: inputs.cgo-enabled == '0' + name: ${{ steps.metadata.outputs.build-step-name }} env: - CGO_ENABLED: ${{ inputs.cgo-enabled }} + CGO_ENABLED: 0 GO_TAGS: ${{ inputs.go-tags }} GOARCH: ${{ inputs.goarch }} GOARM: ${{ inputs.goarm }} @@ -145,6 +125,54 @@ runs: VERSION_METADATA: ${{ inputs.vault-edition != 'ce' && inputs.vault-edition || '' }} shell: bash run: make ci-build + - if: inputs.cgo-enabled == '1' + uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 + with: + driver-opts: network=host # So we can run our own little registry + - if: inputs.cgo-enabled == '1' + shell: bash + run: docker run -d -p 5000:5000 --restart always --name registry registry:2 + - if: inputs.cgo-enabled == '1' + name: Build CGO builder image + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + env: + DOCKER_BUILD_SUMMARY: false + with: + context: . + build-args: | + GO_VERSION=${{ steps.set-up-go.outputs.go-version }} + # Only build a container for the host OS since the same container + # handles cross building. + platforms: linux/amd64 + push: true + target: builder + tags: localhost:5000/vault-builder:${{ steps.metadata.outputs.vault-builder-cache-key }} + # Upload the resulting minimal image to actions cache. This could + # be a problem if the resulting images are too big. + cache-from: type=gha,scope=vault-builder-${{ steps.metadata.outputs.vault-builder-cache-key }} + cache-to: type=gha,mode=min,scope=vault-builder-${{ steps.metadata.outputs.vault-builder-cache-key }} + github-token: ${{ inputs.github-token }} + - if: inputs.cgo-enabled == '1' + name: ${{ steps.metadata.outputs.build-step-name }} + shell: bash + run: | + mkdir -p dist + mkdir -p out + docker run \ + -v $(pwd):/build \ + -v $(go env GOMODCACHE):/go-mod-cache \ + --env GITHUB_TOKEN='${{ inputs.github-token }}' \ + --env CGO_ENABLED=1 \ + --env GO_TAGS='${{ inputs.go-tags }}' \ + --env GOARCH='${{ inputs.goarch }}' \ + --env GOARM='${{ inputs.goarm }}' \ + --env GOEXPERIMENT='${{ inputs.goexperiment }}' \ + --env GOMODCACHE=/go-mod-cache \ + --env GOOS='${{ inputs.goos }}' \ + --env VERSION='${{ inputs.version }}' \ + --env VERSION_METADATA='${{ inputs.vault-edition != 'ce' && inputs.vault-edition || '' }}' \ + localhost:5000/vault-builder:${{ steps.metadata.outputs.vault-builder-cache-key }} \ + make ci-build - if: inputs.vault-edition != 'ce' shell: bash run: make ci-prepare-ent-legal diff --git a/.github/actions/metadata/action.yml b/.github/actions/metadata/action.yml index 6f453bebb6..f19b4a9251 100644 --- a/.github/actions/metadata/action.yml +++ b/.github/actions/metadata/action.yml @@ -24,9 +24,6 @@ outputs: compute-build: description: A JSON encoded "runs-on" for App build worfkflows. value: ${{ steps.workflow-metadata.outputs.compute-build }} - compute-build-compat: - description: A JSON encoded "runs-on" for App build workflows that need an older glibc to link against. - value: ${{ steps.workflow-metadata.outputs.compute-build-compat }} compute-build-ui: description: A JSON encoded "runs-on" for web UI build workflows. value: ${{ steps.workflow-metadata.outputs.compute-build-ui }} @@ -153,7 +150,6 @@ runs: if [ "$is_enterprise" = 'true' ]; then { echo 'compute-build=["self-hosted","ondemand","os=linux","disk_gb=64","type=c6a.4xlarge"]' - echo 'compute-build-compat=["self-hosted","ubuntu-20.04"]' # for older glibc compatibility, m6a.4xlarge echo 'compute-build-ui=["self-hosted","ondemand","os=linux", "disk_gb=64", "type=c6a.2xlarge"]' echo 'compute-test-go=["self-hosted","ondemand","os=linux","disk_gb=64","type=c6a.2xlarge"]' echo 'compute-test-ui=["self-hosted","ondemand","os=linux","type=m6a.2xlarge"]' @@ -165,7 +161,6 @@ runs: else { echo 'compute-build="custom-linux-medium-vault-latest"' - echo 'compute-build-compat="custom-linux-medium-vault-latest"' echo 'compute-build-ui="custom-linux-xl-vault-latest"' echo 'compute-test-go="custom-linux-medium-vault-latest"' echo 'compute-test-ui="custom-linux-medium-vault-latest"' diff --git a/.github/workflows/build-artifacts-ce.yml b/.github/workflows/build-artifacts-ce.yml index 821516ec8e..2c028e1511 100644 --- a/.github/workflows/build-artifacts-ce.yml +++ b/.github/workflows/build-artifacts-ce.yml @@ -23,10 +23,6 @@ on: type: string # JSON encoded to support passing arrays description: A JSON encoded "runs-on" for build worfkflows required: true - compute-build-compat: - type: string # JSON encoded to support passing arrays - description: A JSON encoded "runs-on" for build workflows that need older glibc - required: true compute-small: type: string # JSON encoded to support passing arrays description: A JSON encoded "runs-on" for non-resource-intensive workflows @@ -62,10 +58,6 @@ on: type: string # JSON encoded to support passing arrays description: A JSON encoded "runs-on" for build worfkflows required: true - compute-build-compat: - type: string # JSON encoded to support passing arrays - description: A JSON encoded "runs-on" for build workflows that need older glibc - required: true compute-small: type: string # JSON encoded to support passing arrays description: A JSON encoded "runs-on" for non-resource-intensive workflows diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 48abf643bf..c5c98f80c4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -89,7 +89,6 @@ jobs: changed-files: ${{ steps.changed-files.outputs.changed-files }} checkout-ref: ${{ steps.checkout.outputs.ref }} compute-build: ${{ steps.metadata.outputs.compute-build }} - compute-build-compat: ${{ steps.metadata.outputs.compute-build-compat }} compute-build-ui: ${{ steps.metadata.outputs.compute-build-ui }} compute-small: ${{ steps.metadata.outputs.compute-small }} is-draft: ${{ steps.metadata.outputs.is-draft }} @@ -237,7 +236,6 @@ jobs: build-date: ${{ needs.setup.outputs.build-date }} checkout-ref: ${{ needs.setup.outputs.checkout-ref }} compute-build: ${{ needs.setup.outputs.compute-build }} - compute-build-compat: ${{ needs.setup.outputs.compute-build-compat }} compute-small: ${{ needs.setup.outputs.compute-small }} vault-revision: ${{ needs.setup.outputs.vault-revision }} vault-version: ${{ needs.setup.outputs.vault-version }} diff --git a/Dockerfile b/Dockerfile index 4bdf1b0ce5..3e8eefb1b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -177,3 +177,49 @@ FROM ubi AS ubi-fips FROM ubi AS ubi-hsm FROM ubi AS ubi-hsm-fips + +## Builder: +# +# A build container used to build the Vault binary. We use focal because the +# version of glibc is old enough for all of our supported distros for editions +# that require CGO. +# +# You can build the builder container like so: +# docker build -t builder --build-arg GO_VERSION=$(cat .go-version) . +# +# To can build Vault using the builder container like so: +# docker run -it -v $(pwd):/build -v $(go env GOMODCACHE):/go-mod-cache --env GITHUB_TOKEN=$GITHUB_TOKEN --env GO_TAGS='ui enterprise cgo hsm venthsm' --env GOARCH=s390x --env GOOS=linux --env VERSION=1.20.0-beta1 --env VERSION_METADATA=ent.hsm --env GOMODCACHE=/go-mod-cache --env CGO_ENABLED=1 builder make ci-build +# +# Note that the container is automatically built in CI +FROM ubuntu:focal AS builder + +# Pass in the GO_VERSION as a build-arg +ARG GO_VERSION + +# Set our environment +ENV PATH="/root/go/bin:/opt/go/bin:$PATH" +ENV GOPRIVATE='github.com/hashicorp/*' + +# Install the necessary system tooling to cross compile vault for our various +# CGO targets. Do this separately from branch specific Go and build toolchains +# so our various builder image layers can share cache. +COPY .build/system.sh . +RUN chmod +x system.sh +RUN ./system.sh + +# Install the correct Go toolchain +COPY .build/go.sh . +RUN chmod +x go.sh +RUN ./go.sh + +# Install the vault build tools. Clean up after ourselves so our layer is +# minimal. +COPY tools/tools.sh . +RUN chmod +x tools.sh +RUN ./tools.sh install-external && rm -rf "$(go env GOCACHE)" && rm -rf "$(go env GOMODCACHE)" + +# Run the build +COPY .build/entrypoint.sh . +RUN chmod +x entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/tools/pipeline/internal/pkg/changed/checkers.go b/tools/pipeline/internal/pkg/changed/checkers.go index 6aca0e99fd..21bffdfd2d 100644 --- a/tools/pipeline/internal/pkg/changed/checkers.go +++ b/tools/pipeline/internal/pkg/changed/checkers.go @@ -229,6 +229,7 @@ func FileGroupCheckerPipeline(ctx context.Context, file *File) FileGroups { switch { case + hasBaseDir(name, ".build"), hasBaseDir(name, ".github"), hasBaseDir(name, "scripts"), hasBaseDir(name, filepath.Join("tools", "pipeline")), diff --git a/tools/pipeline/internal/pkg/changed/checkers_test.go b/tools/pipeline/internal/pkg/changed/checkers_test.go index a75c0708b2..9bb14cfcd1 100644 --- a/tools/pipeline/internal/pkg/changed/checkers_test.go +++ b/tools/pipeline/internal/pkg/changed/checkers_test.go @@ -15,6 +15,7 @@ func TestFileGroupDefaultCheckers(t *testing.T) { t.Parallel() for filename, groups := range map[string]FileGroups{ + ".build/entrypoint.sh": {FileGroupPipeline}, ".github/actions/changed-files/actions.yml": {FileGroupPipeline}, ".github/workflows/build.yml": {FileGroupPipeline}, ".github/workflows/build-artifacts-ce.yml": {FileGroupCommunity, FileGroupPipeline},