From befafd5a9c58e8f36a9c954736e1693c67e3f177 Mon Sep 17 00:00:00 2001 From: Ryan Cragun Date: Tue, 3 Jun 2025 17:32:14 -0600 Subject: [PATCH] [VAULT-35682] build(cgo): Build CGO binaries in a container (#30834) Ubuntu 20.04 has reached EOL and is no longer a supported runner host distro. Historically we've relied on it for our CGO builds as it contains an old enough version of glibc that we can retain compatibility with all of our supported distros and build on a single host distro. Rather than requiring a new RHEL 8 builder (or some equivalent), we instead build CGO binaries inside an Ubuntu 20.04 container along with its glibc and various C compilers. I've separated out system package changes, the Go toolchain install, and external build tools tools install into different container layers so that the builder container used for each branch is maximally cacheable. On cache misses these changes result in noticeably longer build times for CGO binaries. That is unavoidable with this strategy. Most of the time our builds will get a cache hit on all layers unless they've changed any of the following: - .build/* - .go-version - .github/actions/build-vault - tools/tools.sh - Dockerfile I've tried my best to reduce the cache space used by each layer. Currently our build container takes about 220MB of cache space. About half of that ought to be shared cache between main and release branches. I would expect total new cache used to be in the 500-600MB range, or about 5% of our total space. Some follow-up idea that we might want to consider: - Build everything inside the build container and remove the github actions that set up external tools - Instead of building external tools with `go install`, migrate them into build scripts that install pre-built `linux/amd64` binaries - Migrate external to `go tool` and use it in the builder container. This requires us to be on 1.24 everywhere so ought not be considered until that is a reality. Signed-off-by: Ryan Cragun --- .build/entrypoint.sh | 45 +++++++ .build/go.sh | 8 ++ .build/system.sh | 41 +++++++ .github/actions/build-vault/action.yml | 112 +++++++++++------- .github/actions/metadata/action.yml | 5 - .github/workflows/build-artifacts-ce.yml | 8 -- .github/workflows/build.yml | 2 - Dockerfile | 46 +++++++ .../pipeline/internal/pkg/changed/checkers.go | 1 + .../internal/pkg/changed/checkers_test.go | 1 + 10 files changed, 212 insertions(+), 57 deletions(-) create mode 100644 .build/entrypoint.sh create mode 100644 .build/go.sh create mode 100644 .build/system.sh diff --git a/.build/entrypoint.sh b/.build/entrypoint.sh new file mode 100644 index 0000000000..a7a7dac306 --- /dev/null +++ b/.build/entrypoint.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +set -e + +fail() { + echo "$1" 1>&2 + exit 1 +} + +[[ -z "$GOARCH" ]] && fail "A GOARCH has not been defined" +[[ -z "$GITHUB_TOKEN" ]] && fail "A GITHUB_TOKEN has not been defined" + +host_arch="$(dpkg --print-architecture)" +host_arch="${host_arch##*-}" +if [[ "$host_arch" != "$GOARCH" ]]; then + # We're building for a different architecture than our target host OS so + # we have to tell the Go compiler to use the correct C cross-compiler for + # our target instead of relying on the host C compiler. + # + # https://packages.ubuntu.com/search?suite=noble§ion=all&arch=any&keywords=linux-gnu-gcc&searchon=contents + case "$GOARCH" in + amd64) + export CC=x86_64-linux-gnu-gcc + ;; + arm64) + export CC=aarch64-linux-gnu-gcc + ;; + s390x) + export CC=s390x-linux-gnu-gcc + ;; + *) + fail "Building for $GOARCH has not been implemented" + ;; + esac +fi + +# Assume that /build is where we've mounted the vault repo. +git config --global --add safe.directory /build +git config --global url."https://${GITHUB_TOKEN}@github.com".insteadOf "https://github.com" + +# Exec our command +cd build || exit 1 +exec "$@" diff --git a/.build/go.sh b/.build/go.sh new file mode 100644 index 0000000000..5790e46509 --- /dev/null +++ b/.build/go.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 +set -e + +host_arch="$(dpkg --print-architecture)" +host_arch="${host_arch##*-}" +curl -L "https://go.dev/dl/go${GO_VERSION}.linux-${host_arch}.tar.gz" | tar -C /opt -zxv diff --git a/.build/system.sh b/.build/system.sh new file mode 100644 index 0000000000..bcb2ff271a --- /dev/null +++ b/.build/system.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +set -e + +export DEBIAN_FRONTEND=noninteractive + +install() { + apt-get install -y "$@" +} + +# Install our cross building tools +# https://packages.ubuntu.com/search?suite=noble§ion=all&arch=any&keywords=crossbuild-essential&searchon=names + +apt-get update +apt-get install -y --no-install-recommends build-essential \ + gcc-s390x-linux-gnu \ + crossbuild-essential-s390x \ + ca-certificates \ + curl \ + git + +host_arch="$(dpkg --print-architecture)" +host_arch="${host_arch##*-}" +case "$host_arch" in + amd64) + install crossbuild-essential-arm64 gcc-aarch64-linux-gnu + ;; + arm64) + install gcc-x86-64-linux-gnu + ;; + *) + echo "Building on $host_arch has not been implemented" 1>&2 + exit 1 + ;; +esac + +# Clean up after ourselves for a minimal image +apt-get clean +rm -rf /var/lib/apt/lists/* diff --git a/.github/actions/build-vault/action.yml b/.github/actions/build-vault/action.yml index 28c884b0a0..af2057be37 100644 --- a/.github/actions/build-vault/action.yml +++ b/.github/actions/build-vault/action.yml @@ -11,59 +11,44 @@ description: | inputs: github-token: - type: string description: An elevated Github token to access private Go modules if necessary. default: "" cgo-enabled: - type: number description: Enable or disable CGO during the build. - default: 0 + default: "0" create-docker-container: - type: boolean description: Package the binary into a Docker/AWS container. - default: true + default: "true" create-redhat-container: - type: boolean description: Package the binary into a Redhat container. - default: false + default: "false" create-packages: - type: boolean description: Package the binaries into deb and rpm formats. - default: true + default: "true" goos: - type: string description: The Go GOOS value environment variable to set during the build. goarch: - type: string description: The Go GOARCH value environment variable to set during the build. goarm: - type: string description: The Go GOARM value environment variable to set during the build. default: "" goexperiment: - type: string description: Which Go experiments to enable. default: "" go-tags: - type: string description: A comma separated list of tags to pass to the Go compiler during build. default: "" package-name: - type: string description: The name to use for the linux packages. default: ${{ github.event.repository.name }} vault-binary-name: - type: string description: The name of the vault binary. default: vault vault-edition: - type: string description: The edition of vault to build. vault-version: - type: string description: The version metadata to inject into the build via the linker. web-ui-cache-key: - type: string description: The cache key for restoring the pre-built web UI artifact. outputs: @@ -74,29 +59,12 @@ outputs: runs: using: composite steps: - - name: Ensure zstd is available for actions/cache - # actions/cache restores based on cache key and "cache version", the former is unique to the - # build job or web UI, the latter is a hash which is based on the runner OS, the paths being - # cached, and the program used to compress it. Most of our workflows will use zstd to compress - # the cached artifact so we have to have it around for our machines to get both a version match - # and to decompress it. Most runners include zstd by default but there are exception like - # our Ubuntu 20.04 compatibility runners which do not. - shell: bash - run: which zstd || (sudo apt update && sudo apt install -y zstd) - - uses: ./.github/actions/set-up-go + - id: set-up-go + uses: ./.github/actions/set-up-go with: github-token: ${{ inputs.github-token }} - - uses: ./.github/actions/install-external-tools - - if: inputs.goarch == 's390x' && inputs.vault-edition == 'ent.hsm' - name: Configure CGO compiler for HSM edition on s390x - shell: bash - run: | - sudo apt-get update - sudo apt-get install -y gcc-multilib-s390x-linux-gnu - { - echo "CC=s390x-linux-gnu-gcc" - echo "CC_FOR_TARGET=s390x-linux-gnu-gcc" - } | tee -a "$GITHUB_ENV" + - if: inputs.cgo-enabled == '0' + uses: ./.github/actions/install-external-tools - if: inputs.vault-edition != 'ce' name: Configure Git shell: bash @@ -126,15 +94,27 @@ runs: build_step_name='Vault ${{ inputs.goos }} ${{ inputs.goarch }} v${{ inputs.vault-version }}+${{ inputs.vault-edition }}' package_version='${{ inputs.vault-version }}+ent' # this should always be +ent here regardless of enterprise edition fi + # Generate a builder cache key that considers anything that might change + # our build container, including: + # - The Go version we're building with + # - External Go build tooling as defined in tools/tools.sh + # - The Dockerfile or .build directory + # - The build-vault Github action + docker_sha=$(git ls-tree HEAD Dockerfile --object-only --abbrev=5) + build_sha=$(git ls-tree HEAD .build --object-only --abbrev=5) + tools_sha=$(git ls-tree HEAD tools/tools.sh --object-only --abbrev=5) + github_sha=$(git ls-tree HEAD .github/actions/build-vault --object-only --abbrev=5) { echo "artifact-basename=$(make ci-get-artifact-basename)" echo "binary-path=dist/${{ inputs.vault-binary-name }}" echo "build-step-name=${build_step_name}" + echo "vault-builder-cache-key=${docker_sha}-${build_sha}-${tools_sha}-${github_sha}-$(cat .go-version)" echo "package-version=${package_version}" } | tee -a "$GITHUB_OUTPUT" - - name: ${{ steps.metadata.outputs.build-step-name }} + - if: inputs.cgo-enabled == '0' + name: ${{ steps.metadata.outputs.build-step-name }} env: - CGO_ENABLED: ${{ inputs.cgo-enabled }} + CGO_ENABLED: 0 GO_TAGS: ${{ inputs.go-tags }} GOARCH: ${{ inputs.goarch }} GOARM: ${{ inputs.goarm }} @@ -145,6 +125,54 @@ runs: VERSION_METADATA: ${{ inputs.vault-edition != 'ce' && inputs.vault-edition || '' }} shell: bash run: make ci-build + - if: inputs.cgo-enabled == '1' + uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 + with: + driver-opts: network=host # So we can run our own little registry + - if: inputs.cgo-enabled == '1' + shell: bash + run: docker run -d -p 5000:5000 --restart always --name registry registry:2 + - if: inputs.cgo-enabled == '1' + name: Build CGO builder image + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + env: + DOCKER_BUILD_SUMMARY: false + with: + context: . + build-args: | + GO_VERSION=${{ steps.set-up-go.outputs.go-version }} + # Only build a container for the host OS since the same container + # handles cross building. + platforms: linux/amd64 + push: true + target: builder + tags: localhost:5000/vault-builder:${{ steps.metadata.outputs.vault-builder-cache-key }} + # Upload the resulting minimal image to actions cache. This could + # be a problem if the resulting images are too big. + cache-from: type=gha,scope=vault-builder-${{ steps.metadata.outputs.vault-builder-cache-key }} + cache-to: type=gha,mode=min,scope=vault-builder-${{ steps.metadata.outputs.vault-builder-cache-key }} + github-token: ${{ inputs.github-token }} + - if: inputs.cgo-enabled == '1' + name: ${{ steps.metadata.outputs.build-step-name }} + shell: bash + run: | + mkdir -p dist + mkdir -p out + docker run \ + -v $(pwd):/build \ + -v $(go env GOMODCACHE):/go-mod-cache \ + --env GITHUB_TOKEN='${{ inputs.github-token }}' \ + --env CGO_ENABLED=1 \ + --env GO_TAGS='${{ inputs.go-tags }}' \ + --env GOARCH='${{ inputs.goarch }}' \ + --env GOARM='${{ inputs.goarm }}' \ + --env GOEXPERIMENT='${{ inputs.goexperiment }}' \ + --env GOMODCACHE=/go-mod-cache \ + --env GOOS='${{ inputs.goos }}' \ + --env VERSION='${{ inputs.version }}' \ + --env VERSION_METADATA='${{ inputs.vault-edition != 'ce' && inputs.vault-edition || '' }}' \ + localhost:5000/vault-builder:${{ steps.metadata.outputs.vault-builder-cache-key }} \ + make ci-build - if: inputs.vault-edition != 'ce' shell: bash run: make ci-prepare-ent-legal diff --git a/.github/actions/metadata/action.yml b/.github/actions/metadata/action.yml index 6f453bebb6..f19b4a9251 100644 --- a/.github/actions/metadata/action.yml +++ b/.github/actions/metadata/action.yml @@ -24,9 +24,6 @@ outputs: compute-build: description: A JSON encoded "runs-on" for App build worfkflows. value: ${{ steps.workflow-metadata.outputs.compute-build }} - compute-build-compat: - description: A JSON encoded "runs-on" for App build workflows that need an older glibc to link against. - value: ${{ steps.workflow-metadata.outputs.compute-build-compat }} compute-build-ui: description: A JSON encoded "runs-on" for web UI build workflows. value: ${{ steps.workflow-metadata.outputs.compute-build-ui }} @@ -153,7 +150,6 @@ runs: if [ "$is_enterprise" = 'true' ]; then { echo 'compute-build=["self-hosted","ondemand","os=linux","disk_gb=64","type=c6a.4xlarge"]' - echo 'compute-build-compat=["self-hosted","ubuntu-20.04"]' # for older glibc compatibility, m6a.4xlarge echo 'compute-build-ui=["self-hosted","ondemand","os=linux", "disk_gb=64", "type=c6a.2xlarge"]' echo 'compute-test-go=["self-hosted","ondemand","os=linux","disk_gb=64","type=c6a.2xlarge"]' echo 'compute-test-ui=["self-hosted","ondemand","os=linux","type=m6a.2xlarge"]' @@ -165,7 +161,6 @@ runs: else { echo 'compute-build="custom-linux-medium-vault-latest"' - echo 'compute-build-compat="custom-linux-medium-vault-latest"' echo 'compute-build-ui="custom-linux-xl-vault-latest"' echo 'compute-test-go="custom-linux-medium-vault-latest"' echo 'compute-test-ui="custom-linux-medium-vault-latest"' diff --git a/.github/workflows/build-artifacts-ce.yml b/.github/workflows/build-artifacts-ce.yml index 821516ec8e..2c028e1511 100644 --- a/.github/workflows/build-artifacts-ce.yml +++ b/.github/workflows/build-artifacts-ce.yml @@ -23,10 +23,6 @@ on: type: string # JSON encoded to support passing arrays description: A JSON encoded "runs-on" for build worfkflows required: true - compute-build-compat: - type: string # JSON encoded to support passing arrays - description: A JSON encoded "runs-on" for build workflows that need older glibc - required: true compute-small: type: string # JSON encoded to support passing arrays description: A JSON encoded "runs-on" for non-resource-intensive workflows @@ -62,10 +58,6 @@ on: type: string # JSON encoded to support passing arrays description: A JSON encoded "runs-on" for build worfkflows required: true - compute-build-compat: - type: string # JSON encoded to support passing arrays - description: A JSON encoded "runs-on" for build workflows that need older glibc - required: true compute-small: type: string # JSON encoded to support passing arrays description: A JSON encoded "runs-on" for non-resource-intensive workflows diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 48abf643bf..c5c98f80c4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -89,7 +89,6 @@ jobs: changed-files: ${{ steps.changed-files.outputs.changed-files }} checkout-ref: ${{ steps.checkout.outputs.ref }} compute-build: ${{ steps.metadata.outputs.compute-build }} - compute-build-compat: ${{ steps.metadata.outputs.compute-build-compat }} compute-build-ui: ${{ steps.metadata.outputs.compute-build-ui }} compute-small: ${{ steps.metadata.outputs.compute-small }} is-draft: ${{ steps.metadata.outputs.is-draft }} @@ -237,7 +236,6 @@ jobs: build-date: ${{ needs.setup.outputs.build-date }} checkout-ref: ${{ needs.setup.outputs.checkout-ref }} compute-build: ${{ needs.setup.outputs.compute-build }} - compute-build-compat: ${{ needs.setup.outputs.compute-build-compat }} compute-small: ${{ needs.setup.outputs.compute-small }} vault-revision: ${{ needs.setup.outputs.vault-revision }} vault-version: ${{ needs.setup.outputs.vault-version }} diff --git a/Dockerfile b/Dockerfile index 4bdf1b0ce5..3e8eefb1b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -177,3 +177,49 @@ FROM ubi AS ubi-fips FROM ubi AS ubi-hsm FROM ubi AS ubi-hsm-fips + +## Builder: +# +# A build container used to build the Vault binary. We use focal because the +# version of glibc is old enough for all of our supported distros for editions +# that require CGO. +# +# You can build the builder container like so: +# docker build -t builder --build-arg GO_VERSION=$(cat .go-version) . +# +# To can build Vault using the builder container like so: +# docker run -it -v $(pwd):/build -v $(go env GOMODCACHE):/go-mod-cache --env GITHUB_TOKEN=$GITHUB_TOKEN --env GO_TAGS='ui enterprise cgo hsm venthsm' --env GOARCH=s390x --env GOOS=linux --env VERSION=1.20.0-beta1 --env VERSION_METADATA=ent.hsm --env GOMODCACHE=/go-mod-cache --env CGO_ENABLED=1 builder make ci-build +# +# Note that the container is automatically built in CI +FROM ubuntu:focal AS builder + +# Pass in the GO_VERSION as a build-arg +ARG GO_VERSION + +# Set our environment +ENV PATH="/root/go/bin:/opt/go/bin:$PATH" +ENV GOPRIVATE='github.com/hashicorp/*' + +# Install the necessary system tooling to cross compile vault for our various +# CGO targets. Do this separately from branch specific Go and build toolchains +# so our various builder image layers can share cache. +COPY .build/system.sh . +RUN chmod +x system.sh +RUN ./system.sh + +# Install the correct Go toolchain +COPY .build/go.sh . +RUN chmod +x go.sh +RUN ./go.sh + +# Install the vault build tools. Clean up after ourselves so our layer is +# minimal. +COPY tools/tools.sh . +RUN chmod +x tools.sh +RUN ./tools.sh install-external && rm -rf "$(go env GOCACHE)" && rm -rf "$(go env GOMODCACHE)" + +# Run the build +COPY .build/entrypoint.sh . +RUN chmod +x entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/tools/pipeline/internal/pkg/changed/checkers.go b/tools/pipeline/internal/pkg/changed/checkers.go index 6aca0e99fd..21bffdfd2d 100644 --- a/tools/pipeline/internal/pkg/changed/checkers.go +++ b/tools/pipeline/internal/pkg/changed/checkers.go @@ -229,6 +229,7 @@ func FileGroupCheckerPipeline(ctx context.Context, file *File) FileGroups { switch { case + hasBaseDir(name, ".build"), hasBaseDir(name, ".github"), hasBaseDir(name, "scripts"), hasBaseDir(name, filepath.Join("tools", "pipeline")), diff --git a/tools/pipeline/internal/pkg/changed/checkers_test.go b/tools/pipeline/internal/pkg/changed/checkers_test.go index a75c0708b2..9bb14cfcd1 100644 --- a/tools/pipeline/internal/pkg/changed/checkers_test.go +++ b/tools/pipeline/internal/pkg/changed/checkers_test.go @@ -15,6 +15,7 @@ func TestFileGroupDefaultCheckers(t *testing.T) { t.Parallel() for filename, groups := range map[string]FileGroups{ + ".build/entrypoint.sh": {FileGroupPipeline}, ".github/actions/changed-files/actions.yml": {FileGroupPipeline}, ".github/workflows/build.yml": {FileGroupPipeline}, ".github/workflows/build-artifacts-ce.yml": {FileGroupCommunity, FileGroupPipeline},